Merge pull request #629 from OpenKnowledgeMaps/base-query-sanitization

Base query sanitization

Author: chreman

server/preprocessing/other-scripts/base.R

@@ -140,6 +140,8 @@ get_papers <- function(query, params, limit=100,
   subject_cleaned = gsub(" ?\\d[:?-?]?(\\d+.)+", "", subject_cleaned) # replace residuals like 5:621.313.323 or '5-76.95'
   subject_cleaned = gsub("\\w+:\\w+-(\\w+\\/)+", "", subject_cleaned) # replace residuals like Info:eu-repo/classification/
   subject_cleaned = gsub("^; $", "", subject_cleaned) # replace residuals like Info:eu-repo/classification/
+  subject_cleaned = gsub(",", ", ", subject_cleaned) # clean up keyword separation
+  subject_cleaned = gsub("\\s+", " ", subject_cleaned) # clean up keyword separation
 
   metadata$subject = subject_cleaned
 

server/preprocessing/other-scripts/utils.R

@@ -2,6 +2,7 @@ library(stringdist)
 library(logging)
 
 sanitize_query <- function(query) {
+  query <- gsub("\\", "", query, fixed=T)
   sanitized_query <- gsub('[“”]', '"', query)
   return(list(raw_query=query, sanitized_query=sanitized_query))
 }