various bugfixes

Author: chreman

server/preprocessing/other-scripts/base.R

@@ -189,7 +189,7 @@ etl <- function(res, repo, non_public) {
   subject_cleaned = gsub("[^:;]+ ?:: ?[^;]+(;|$)?", "", subject_cleaned) #remove classification with separator ::
   subject_cleaned = gsub("[^\\[;]+\\[[A-Z,0-9]+\\](;|$)?", "", subject_cleaned) # remove WHO classification
   subject_cleaned = gsub("Info:\\w+-(\\w+\\/)+", "", subject_cleaned) # remove Info:eu-repo/classification/
-  subject_cleaned = gsub("([A-Za-z]+:[A-Za-z0-9][A-Za-z0-9 \\/\\.]+);?", "", subject_cleaned, perl=TRUE) # clean up annotations with prefix e.g. theme:annotation
+  subject_cleaned = gsub("([A-Za-z]+:[A-Za-z0-9 \\/\\.-]+);?", "", subject_cleaned, perl=TRUE) # clean up annotations with prefix e.g. theme:annotation
   if (!is.null(params$vis_type) && params$vis_type == "timeline") {
     subject_cleaned = gsub("FOS ", "", subject_cleaned) # remove FOS classification tag, but keep classifcation name
     arxiv_classification_string = "(cs|econ|eess|math|astro-ph|nlin|q-bio|q-fin|stat)\\.[A-Z]{2}|cond-mat\\.[a-z\\-]+|hep-(ex|lat|ph|th)|math-ph|nucl-(ex|th)|physics\\.[a-z\\-]+|(astro-ph|gr-qc|quant-ph|cond-mat)"
@@ -215,6 +215,7 @@ etl <- function(res, repo, non_public) {
   subject_cleaned = gsub(": ", "", subject_cleaned) # clean up keyword separation
   subject_cleaned = gsub("^; $", "", subject_cleaned) # clean up keyword separation
   subject_cleaned = gsub(";+", ";", subject_cleaned) # clean up keyword separation
+  subject_cleaned = gsub(",+", ",", subject_cleaned) # clean up keyword separation
   subject_cleaned = gsub(",", ", ", subject_cleaned) # clean up keyword separation
   subject_cleaned = gsub("\\s+", " ", subject_cleaned) # clean up keyword separation
   subject_cleaned = stringi::stri_trim(subject_cleaned) # clean up keyword separation

server/preprocessing/other-scripts/summarize.R

@@ -97,7 +97,9 @@ create_cluster_labels <- function(clusters, metadata,
 
 fix_cluster_labels <- function(clusterlabels, type_counts){
   unlist(mclapply(clusterlabels, function(x) {
-    fix_keyword_casing(x, type_counts)
+    x <- fix_keyword_casing(x, type_counts)
+    # clean up titles from format issues
+    x <- gsub(",+", ",", x)
     }))
 }
 

server/workers/dataprocessing/src/headstart.py

@@ -92,8 +92,6 @@ def run(self):
                 self.logger.error("Could not connect to remote Redis server, is the SSH tunnel open?")
             try:
                 if params.get('vis_type') == "timeline":
-                    # the step of create_map can be dropped once deduplication is possible in API backend as well
-                    # TODO: create deduplicate endpoint in service worker and connect to that
                     metadata = self.execute_search(params, input_data)
                     sg_data = sg.get_streamgraph_data(json.loads(metadata),
                                                     params.get('q'),