Merge pull request #869 from OpenKnowledgeMaps/feat/remove-mesh-keywords-in-streamgraph

feat: remove mesh keywords in streamgraph

Author: andreishket

server/preprocessing/other-scripts/base.R

@@ -248,7 +248,13 @@ etl <- function(res, repo, non_public) {
   subject_cleaned = gsub("(wikidata)?\\.org/entity/[qQ]([\\d]+)?", "", subject_cleaned) # remove wikidata classification
   subject_cleaned = gsub("</keyword><keyword>", "", subject_cleaned) # remove </keyword><keyword>
   subject_cleaned = gsub("\\[No keyword\\]", "", subject_cleaned)
-  subject_cleaned = gsub("\\[[^]]*\\]", "", subject_cleaned) # remove any text inside square brackets
+
+  if (!is.null(params$vis_type) && params$vis_type == "timeline") {
+    subject_cleaned = remove_keywords_with_text_in_square_brackets(subject_cleaned)
+  } else {
+    subject_cleaned = remove_text_in_square_brackets_from_keywords(subject_cleaned)
+  }
+
   subject_cleaned = gsub("\\[[^\\[]+\\][^\\;]+(;|$)?", "", subject_cleaned) # remove classification
   subject_cleaned = gsub("[0-9]{2,} [A-Z]+[^;]*(;|$)?", "", subject_cleaned) #remove classification
   subject_cleaned = gsub(" -- ", "; ", subject_cleaned) #replace inconsistent keyword separation
@@ -357,6 +363,18 @@ decode_dctypenorm <- function(dctypestring) {
   return(typecodes)
 }
 
+remove_keywords_with_text_in_square_brackets <- function(x) {
+  # This function removes whole keywords that contain text in square brackets.
+  # Example: 'Climate [MeSH]' | 'Some keywords [Chemical]'.
+  gsub("[^;]*\\[[^]]+\\][^;]*;?", "", x)
+}
+
+remove_text_in_square_brackets_from_keywords <- function(x) {
+  # This function removes text in square brackets.
+  # Example: 'Climate [MeSH]' -> 'Climate'| 'Some keywords [Chemical]' -> 'Some keywords'.
+  gsub("\\[[^]]*\\]", "", x)
+}
+
 dctypenorm_decoder <- list(
   "4"="Audio",
   "11"="Book",

server/workers/common/common/contentproviders.json

@@ -1,4 +1,68 @@
 [
+  {
+    "name": "VMRF Digital Repository",
+    "internal_name": "ftid14684"
+  },
+  {
+    "name": "STA Dergi",
+    "internal_name": "ftid14686"
+  },
+  {
+    "name": "CityUHK Scholars",
+    "internal_name": "ftid14687"
+  },
+  {
+    "name": "Evolutio Press",
+    "internal_name": "ftid14688"
+  },
+  {
+    "name": "Perry Research",
+    "internal_name": "ftid14689"
+  },
+  {
+    "name": "University of Mohamed Boudiaf - M'Sila",
+    "internal_name": "ftid14690"
+  },
+  {
+    "name": "Mekelle University Institutional Repository",
+    "internal_name": "ftid14695"
+  },
+  {
+    "name": "Atlas social de France",
+    "internal_name": "ftid14685"
+  },
+  {
+    "name": "Sustainable Trends and Business Research (STBR)",
+    "internal_name": "ftid14672"
+  },
+  {
+    "name": "Scripta Intelektual",
+    "internal_name": "ftid14674"
+  },
+  {
+    "name": "Proceedings Centre-Mersenne",
+    "internal_name": "ftid14678"
+  },
+  {
+    "name": "United Journal of Chemistry",
+    "internal_name": "crid14680"
+  },
+  {
+    "name": "University of Biskra Journals",
+    "internal_name": "ftid14681"
+  },
+  {
+    "name": "Computational and Applied Science Journal (CAS Journal)",
+    "internal_name": "ftid14692"
+  },
+  {
+    "name": "International Journal of Research Development and Technology (IJRDT)",
+    "internal_name": "ftid14694"
+  },
+  {
+    "name": "Innovative Science and Technology Publishers",
+    "internal_name": "ftid14696"
+  },
   {
     "name": "Law and innovations",
     "internal_name": "ftid14669"