Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2026-02-16 11:36:53 +01:00
Code Issues Packages Projects Releases Activity

Merge pull request #1313 from scm-manager/bugfix/reduce_logging

Browse Source 
Fix logging of large stacktrace for unknown language
This commit is contained in:
Konstantin Schaper
2020-09-01 11:55:17 +02:00
committed by GitHub
parent 35aaf4dcdd ba15f0579e
commit 61da1c1b62
3 changed files with 161 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- JWT token timeout is now handled properly ([#1297](https://github.com/scm-manager/scm-manager/pull/1297))
- Fix text-overflow in danger zone ([#1298](https://github.com/scm-manager/scm-manager/pull/1298))
- Fix plugin installation error if previously a plugin was installed with the same dependency which is still pending. ([#1300](https://github.com/scm-manager/scm-manager/pull/1300))
- Fix logging of large stacktrace for unknown language ([#1313](https://github.com/scm-manager/scm-manager/pull/1313))
- Remove obsolete revision encoding on sources ([#1315](https://github.com/scm-manager/scm-manager/pull/1315))
## [2.4.0] - 2020-08-14

148
scm-webapp/src/main/java/sonia/scm/web/i18n/I18nServlet.java
View File

@@ -21,7 +21,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.web.i18n;
@@ -31,14 +31,14 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
import sonia.scm.NotFoundException;
import sonia.scm.SCMContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.SCMContextProvider;
import sonia.scm.Stage;
import sonia.scm.lifecycle.RestartEvent;
import sonia.scm.cache.Cache;
import sonia.scm.cache.CacheManager;
import sonia.scm.filter.WebElement;
import sonia.scm.lifecycle.RestartEvent;
import sonia.scm.plugin.PluginLoader;
import javax.inject.Inject;
@@ -51,11 +51,6 @@ import java.net.URL;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.NotFoundException.notFound;
/**
@@ -63,122 +58,109 @@ import static sonia.scm.NotFoundException.notFound;
*/
@Singleton
@WebElement(value = I18nServlet.PATTERN, regex = true)
@Slf4j
public class I18nServlet extends HttpServlet {
private static final Logger LOG = LoggerFactory.getLogger(I18nServlet.class);
public static final String PLUGINS_JSON = "plugins.json";
public static final String PATTERN = "/locales/[a-z\\-A-Z]*/" + PLUGINS_JSON;
public static final String CACHE_NAME = "sonia.cache.plugins.translations";
private final SCMContextProvider context;
private final ClassLoader classLoader;
private final Cache<String, JsonNode> cache;
private static ObjectMapper objectMapper = new ObjectMapper();
private final ObjectMapper objectMapper = new ObjectMapper();
@Inject
public I18nServlet(PluginLoader pluginLoader, CacheManager cacheManager) {
public I18nServlet(SCMContextProvider context, PluginLoader pluginLoader, CacheManager cacheManager) {
this.context = context;
this.classLoader = pluginLoader.getUberClassLoader();
this.cache = cacheManager.getCache(CACHE_NAME);
}
@Subscribe(async = false)
public void handleRestartEvent(RestartEvent event) {
log.debug("Clear cache on restart event with reason {}", event.getReason());
LOG.debug("Clear cache on restart event with reason {}", event.getReason());
cache.clear();
}
private JsonNode getCollectedJson(String path,
Function<String, Optional<JsonNode>> jsonFileProvider,
BiConsumer<String, JsonNode> createdJsonFileConsumer) {
return Optional.ofNullable(jsonFileProvider.apply(path)
.orElseGet(() -> {
Optional<JsonNode> createdFile = collectJsonFile(path);
createdFile.ifPresent(map -> createdJsonFileConsumer.accept(path, map));
return createdFile.orElse(null);
}
)).orElseThrow(() -> notFound(entity("jsonprovider", path)));
}
@VisibleForTesting
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse response) {
protected void doGet(HttpServletRequest request, HttpServletResponse response) {
String path = request.getServletPath();
try {
Optional<JsonNode> json = findJson(path);
if (json.isPresent()) {
write(response, json.get());
} else {
LOG.debug("could not find translation at {}", path);
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
}
} catch (IOException ex) {
LOG.error("Error on getting the translation of the plugins", ex);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}
private void write(HttpServletResponse response, JsonNode jsonNode) throws IOException {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
response.setHeader("Cache-Control", "no-cache");
try (PrintWriter out = response.getWriter()) {
String path = req.getServletPath();
Function<String, Optional<JsonNode>> jsonFileProvider = usedPath -> Optional.empty();
BiConsumer<String, JsonNode> createdJsonFileConsumer = (usedPath, jsonNode) -> log.debug("A json File is created from the path {}", usedPath);
if (isProductionStage()) {
log.debug("In Production Stage get the plugin translations from the cache");
jsonFileProvider = usedPath -> Optional.ofNullable(
cache.get(usedPath));
createdJsonFileConsumer = createdJsonFileConsumer
.andThen((usedPath, jsonNode) -> log.debug("Put the created json File in the cache with the key {}", usedPath))
.andThen(cache::put);
}
objectMapper.writeValue(out, getCollectedJson(path, jsonFileProvider, createdJsonFileConsumer));
} catch (IOException e) {
log.error("Error on getting the translation of the plugins", e);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
} catch (NotFoundException e) {
log.error("Plugin translations are not found", e);
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
try (PrintWriter writer = response.getWriter()) {
objectMapper.writeValue(writer, jsonNode);
}
}
public Optional<JsonNode> findJson(String path) throws IOException {
if (isProductionStage()) {
return findJsonCached(path);
}
return collectJsonFile(path);
}
private Optional<JsonNode> findJsonCached(String path) throws IOException {
JsonNode jsonNode = cache.get(path);
if (jsonNode != null) {
LOG.debug("return json node from cache for path {}", path);
return Optional.of(jsonNode);
}
LOG.debug("collect json for path {}", path);
Optional<JsonNode> collected = collectJsonFile(path);
collected.ifPresent(node -> cache.put(path, node));
return collected;
}
@VisibleForTesting
protected boolean isProductionStage() {
return SCMContext.getContext().getStage() == Stage.PRODUCTION;
return context.getStage() == Stage.PRODUCTION;
}
/**
* Return a collected Json File as JsonNode from the given path from all plugins in the class path
*
* @param path the searched resource path
* @return a collected Json File as JsonNode from the given path from all plugins in the class path
*/
@VisibleForTesting
protected Optional<JsonNode> collectJsonFile(String path) {
log.debug("Collect plugin translations from path {} for every plugin", path);
private Optional<JsonNode> collectJsonFile(String path) throws IOException {
LOG.debug("Collect plugin translations from path {} for every plugin", path);
JsonNode mergedJsonNode = null;
try {
Enumeration<URL> resources = classLoader.getResources(path.replaceFirst("/", ""));
while (resources.hasMoreElements()) {
URL url = resources.nextElement();
JsonNode jsonNode = objectMapper.readTree(url);
if (mergedJsonNode != null) {
merge(mergedJsonNode, jsonNode);
} else {
mergedJsonNode = jsonNode;
}
Enumeration<URL> resources = classLoader.getResources(path.replaceFirst("/", ""));
while (resources.hasMoreElements()) {
URL url = resources.nextElement();
JsonNode jsonNode = objectMapper.readTree(url);
if (mergedJsonNode != null) {
merge(mergedJsonNode, jsonNode);
} else {
mergedJsonNode = jsonNode;
}
} catch (IOException e) {
log.error("Error on loading sources from {}", path, e);
return Optional.empty();
}
return Optional.ofNullable(mergedJsonNode);
}
/**
* Merge the <code>updateNode</code> into the <code>mainNode</code> and return it.
*
* This is not a deep merge.
*
* @param mainNode the main node
* @param updateNode the update node
* @return the merged mainNode
*/
@VisibleForTesting
protected JsonNode merge(JsonNode mainNode, JsonNode updateNode) {
private JsonNode merge(JsonNode mainNode, JsonNode updateNode) {
Iterator<String> fieldNames = updateNode.fieldNames();
while (fieldNames.hasNext()) {
String fieldName = fieldNames.next();
JsonNode jsonNode = mainNode.get(fieldName);
if (jsonNode != null) {
mergeNode(updateNode, fieldName, jsonNode);
} else {

217
scm-webapp/src/test/java/sonia/scm/web/i18n/I18nServletTest.java
View File

@@ -21,49 +21,43 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.web.i18n;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import com.github.legman.EventBus;
import org.apache.commons.lang3.StringUtils;
import org.assertj.core.util.Lists;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.MockSettings;
import org.mockito.internal.creation.MockSettingsImpl;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import sonia.scm.lifecycle.RestartEvent;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.SCMContextProvider;
import sonia.scm.Stage;
import sonia.scm.cache.Cache;
import sonia.scm.cache.CacheManager;
import sonia.scm.event.ScmEventBus;
import sonia.scm.lifecycle.RestartEventFactory;
import sonia.scm.plugin.PluginLoader;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.*;
@RunWith(MockitoJUnitRunner.Silent.class)
public class I18nServletTest {
@ExtendWith(MockitoExtension.class)
class I18nServletTest {
private static final String GIT_PLUGIN_JSON = json(
"{",
@@ -76,6 +70,7 @@ public class I18nServletTest {
"}",
"}"
);
private static final String HG_PLUGIN_JSON = json(
"{",
"'scm-hg-plugin': {",
@@ -87,7 +82,8 @@ public class I18nServletTest {
"}",
"}"
);
private static String SVN_PLUGIN_JSON = json(
private static final String SVN_PLUGIN_JSON = json(
"{",
"'scm-svn-plugin': {",
"'information': {",
@@ -101,59 +97,49 @@ public class I18nServletTest {
return String.join("\n", parts ).replaceAll("'", "\"");
}
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Mock
private SCMContextProvider context;
@Mock
private PluginLoader pluginLoader;
@Mock
private ClassLoader classLoader;
@Mock
private CacheManager cacheManager;
@Mock
private ClassLoader classLoader;
private Cache<String, JsonNode> cache;
private I18nServlet servlet;
@Mock
private Cache cache;
private Enumeration<URL> resources;
@Before
@SuppressWarnings("unchecked")
public void init() throws IOException {
resources = Collections.enumeration(Lists.newArrayList(
createFileFromString(SVN_PLUGIN_JSON).toURI().toURL(),
createFileFromString(GIT_PLUGIN_JSON).toURI().toURL(),
createFileFromString(HG_PLUGIN_JSON).toURI().toURL()
));
@BeforeEach
void init() {
when(pluginLoader.getUberClassLoader()).thenReturn(classLoader);
when(cacheManager.getCache(I18nServlet.CACHE_NAME)).thenReturn(cache);
MockSettings settings = new MockSettingsImpl<>();
settings.useConstructor(pluginLoader, cacheManager);
settings.defaultAnswer(InvocationOnMock::callRealMethod);
servlet = mock(I18nServlet.class, settings);
when(cacheManager.<String, JsonNode>getCache(I18nServlet.CACHE_NAME)).thenReturn(cache);
servlet = new I18nServlet(context, pluginLoader, cacheManager);
}
@Test
public void shouldCleanCacheOnRestartEvent() {
ScmEventBus.getInstance().register(servlet);
ScmEventBus.getInstance().post(RestartEventFactory.create(I18nServlet.class, "Restart to reload the plugin resources"));
void shouldCleanCacheOnRestartEvent() {
EventBus eventBus = new EventBus("forTestingOnly");
eventBus.register(servlet);
eventBus.post(RestartEventFactory.create(I18nServlet.class, "Restart to reload the plugin resources"));
verify(cache).clear();
}
@Test
@SuppressWarnings("unchecked")
public void shouldFailWith404OnMissingResources() throws IOException {
void shouldFailWith404OnMissingResources() throws IOException {
String path = "/locales/de/plugins.json";
HttpServletRequest request = mock(HttpServletRequest.class);
HttpServletResponse response = mock(HttpServletResponse.class);
PrintWriter writer = mock(PrintWriter.class);
when(response.getWriter()).thenReturn(writer);
when(request.getServletPath()).thenReturn(path);
when(classLoader.getResources("locales/de/plugins.json")).thenThrow(IOException.class);
when(classLoader.getResources("locales/de/plugins.json")).thenReturn(
I18nServlet.class.getClassLoader().getResources("something/not/available")
);
servlet.doGet(request, response);
@@ -161,98 +147,96 @@ public class I18nServletTest {
}
@Test
@SuppressWarnings("unchecked")
public void shouldFailWith500OnIOException() throws IOException {
void shouldFailWith500OnIOException() throws IOException {
stage(Stage.DEVELOPMENT);
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getServletPath()).thenReturn("/locales/de/plugins.json");
HttpServletResponse response = mock(HttpServletResponse.class);
doThrow(IOException.class).when(response).getWriter();
when(classLoader.getResources("locales/de/plugins.json")).thenThrow(new IOException("failed"));
servlet.doGet(request, response);
verify(response).setStatus(500);
}
private void stage(Stage stage) {
when(context.getStage()).thenReturn(stage);
}
@Test
@SuppressWarnings("unchecked")
public void inDevelopmentStageShouldNotUseCache() throws IOException {
String path = "/locales/de/plugins.json";
when(servlet.isProductionStage()).thenReturn(false);
void inDevelopmentStageShouldNotUseCache(@TempDir Path temp) throws IOException {
stage(Stage.DEVELOPMENT);
mockResources(temp, "locales/de/plugins.json");
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getServletPath()).thenReturn("/locales/de/plugins.json");
HttpServletResponse response = mock(HttpServletResponse.class);
File file = temporaryFolder.newFile();
PrintWriter writer = new PrintWriter(new FileOutputStream(file));
when(response.getWriter()).thenReturn(writer);
when(request.getServletPath()).thenReturn(path);
when(classLoader.getResources("locales/de/plugins.json")).thenReturn(resources);
String json = doGetString(request, response);
servlet.doGet(request, response);
String json = Files.readLines(file, Charset.defaultCharset()).get(0);
assertJson(json);
verify(cache, never()).get(any());
}
@Test
@SuppressWarnings("unchecked")
public void inProductionStageShouldUseCache() throws IOException {
String path = "/locales/de/plugins.json";
when(servlet.isProductionStage()).thenReturn(true);
HttpServletRequest request = mock(HttpServletRequest.class);
HttpServletResponse response = mock(HttpServletResponse.class);
File file = temporaryFolder.newFile();
PrintWriter writer = new PrintWriter(new FileOutputStream(file));
private String doGetString(HttpServletRequest request, HttpServletResponse response) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(baos);
when(response.getWriter()).thenReturn(writer);
when(request.getServletPath()).thenReturn(path);
when(classLoader.getResources("locales/de/plugins.json")).thenReturn(resources);
servlet.doGet(request, response);
String json = Files.readLines(file, Charset.defaultCharset()).get(0);
assertJson(json);
verify(cache).get(path);
verify(cache).put(eq(path), any());
writer.flush();
return baos.toString(StandardCharsets.UTF_8.name());
}
private void mockResources(Path directory, String resourcePath) throws IOException {
Enumeration<URL> resources = Collections.enumeration(
Arrays.asList(
toURL(directory, "git.json", GIT_PLUGIN_JSON),
toURL(directory, "hg.json", HG_PLUGIN_JSON),
toURL(directory, "svn.json", SVN_PLUGIN_JSON)
)
);
when(classLoader.getResources(resourcePath)).thenReturn(resources);
}
private URL toURL(Path directory, String name, String content) throws IOException {
Path file = directory.resolve(name);
java.nio.file.Files.write(file, content.getBytes(StandardCharsets.UTF_8));
return file.toUri().toURL();
}
@Test
void shouldGetFromCacheInProductionStage() throws IOException {
String path = "/locales/de/plugins.json";
stage(Stage.PRODUCTION);
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getServletPath()).thenReturn(path);
HttpServletResponse response = mock(HttpServletResponse.class);
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(GIT_PLUGIN_JSON);
when(cache.get(path)).thenReturn(jsonNode);
String json = doGetString(request, response);
assertThat(json).contains("scm-git-plugin").doesNotContain("scm-hg-plugin");
verifyHeaders(response);
}
@Test
@SuppressWarnings("unchecked")
public void inProductionStageShouldGetFromCache() throws IOException {
void shouldStoreToCacheInProductionStage(@TempDir Path temp) throws IOException {
String path = "/locales/de/plugins.json";
when(servlet.isProductionStage()).thenReturn(true);
mockResources(temp, "locales/de/plugins.json");
stage(Stage.PRODUCTION);
HttpServletRequest request = mock(HttpServletRequest.class);
HttpServletResponse response = mock(HttpServletResponse.class);
File file = temporaryFolder.newFile();
PrintWriter writer = new PrintWriter(new FileOutputStream(file));
when(response.getWriter()).thenReturn(writer);
when(request.getServletPath()).thenReturn(path);
when(classLoader.getResources("locales/de/plugins.json")).thenReturn(resources);
ObjectMapper objectMapper = new ObjectMapper();
JsonNode node = objectMapper.readTree(GIT_PLUGIN_JSON);
node = servlet.merge(node, objectMapper.readTree(HG_PLUGIN_JSON));
node = servlet.merge(node, objectMapper.readTree(SVN_PLUGIN_JSON));
when(cache.get(path)).thenReturn(node);
HttpServletResponse response = mock(HttpServletResponse.class);
servlet.doGet(request, response);
String json = doGetString(request, response);
String json = Files.readLines(file, Charset.defaultCharset()).get(0);
verify(servlet, never()).collectJsonFile(path);
verify(cache, never()).put(eq(path), any());
verify(cache).get(path);
assertJson(json);
verify(cache).put(any(String.class), any(JsonNode.class));
verifyHeaders(response);
}
@Test
@SuppressWarnings("unchecked")
public void shouldCollectJsonFile() throws IOException {
String path = "locales/de/plugins.json";
when(classLoader.getResources(path)).thenReturn(resources);
Optional<JsonNode> jsonNodeOptional = servlet.collectJsonFile("/" + path);
assertJson(jsonNodeOptional.orElse(null));
assertJson(json);
}
private void verifyHeaders(HttpServletResponse response) {
@@ -261,22 +245,11 @@ public class I18nServletTest {
verify(response).setHeader("Cache-Control", "no-cache");
}
public void assertJson(JsonNode actual) throws IOException {
assertJson(actual.toString());
}
private void assertJson(String actual) throws IOException {
private void assertJson(String actual) {
assertThat(actual)
.isNotEmpty()
.contains(StringUtils.deleteWhitespace(GIT_PLUGIN_JSON.substring(1, GIT_PLUGIN_JSON.length() - 1)))
.contains(StringUtils.deleteWhitespace(HG_PLUGIN_JSON.substring(1, HG_PLUGIN_JSON.length() - 1)))
.contains(StringUtils.deleteWhitespace(SVN_PLUGIN_JSON.substring(1, SVN_PLUGIN_JSON.length() - 1)));
}
private File createFileFromString(String json) throws IOException {
File file = temporaryFolder.newFile();
Files.write(json.getBytes(Charsets.UTF_8), file);
return file;
}
}