Merge branch 'dev' into fontawesome-update

Author: Konstiman

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 2022-01-20
+
+### New features
+
+- backend data sanitization
+- pdfs searchable by the searchbox
+
+### Changes
+
+- data preprocessing refactoring
+
+### Bug fixes
+
+- footer size on narrow screens
+
+### Internal
+
+- test database update
+
+## 2021-12-22
+
+### Changes
+
+- unified HTML & CSS of various paper tags (access tags, dataset tag, custom tags)
+
 ## 2021-12-09
 
 ### New features

server/storage/test.sqlite

@@ -267,9 +267,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         ger: {
             loading: "Wird geladen...",
@@ -340,9 +337,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         ger_linkedcat: {
            loading: "Wird geladen...",
@@ -416,9 +410,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         eng_plos: {
             loading: "Loading...",
@@ -481,9 +472,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         eng_pubmed: {
             loading: "Loading knowledge map.",
@@ -567,9 +555,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         eng_openaire: {
             loading: "Loading...",
@@ -659,9 +644,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
          ger_cris: {
             loading: "Wird geladen...",
@@ -725,9 +707,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         ger_cris_2: {
             loading: "Wird geladen...",
@@ -791,9 +770,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         eng_cris_2: {
            loading: "Loading...",
@@ -861,9 +837,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
         eng_gsheets: {
             loading: "Updating and retrieving map. This may take a few seconds, please hold on.",
@@ -945,9 +918,6 @@ var config = {
             cite_vis_km: "Please cite this knowledge map as follows",
             cite_vis_sg: "Please cite this streamgraph as follows",
             cite_paper: "Cite this document as",
-            cite_metadata_warn_1: "Unfortunately important metadata is missing",
-            cite_metadata_warn_2: "for this document (e.g. title, authors, year). Therefore we are unable to provide meaningful citations. If you'd like to find out more about this document, please visit the",
-            cite_metadata_warn_3: "document's website",
         },
     },
 

vis/js/default-config.js

@@ -24,9 +24,6 @@ const Details = ({ authors, source, isSelected }) => {
       </div>
       {!!source && (
         <div className={"list_source" + (isSelected ? "" : " short")}>
-          <span className="list_in">
-            <Highlight> in </Highlight>
-          </span>
           <span className="list_published_in">
             <Highlight queryHighlight>{source}</Highlight>
           </span>

vis/js/templates/listentry/Details.jsx

@@ -4,6 +4,7 @@ import { connect } from "react-redux";
 import useMatomo from "../../utils/useMatomo";
 import { getPaperPDFClickHandler } from "../../utils/data";
 import { mapDispatchToListEntriesProps } from "../../utils/eventhandlers";
+import Highlight from "../../components/Highlight";
 
 const PaperButtons = ({
   paper,
@@ -29,8 +30,8 @@ const PaperButtons = ({
     // html template starts here
     <div className="paper_buttons_row">
       {!!onPDFClick && (
-        <button className="paper_button" onClick={handlePDFButtonClick}>
-          <i className="far fa-file-pdf"></i>&nbsp;&nbsp;PDF
+        <button className="paper_button main" onClick={handlePDFButtonClick}>
+          <i className="far fa-file-pdf"></i>&nbsp;&nbsp;<Highlight>PDF</Highlight>
         </button>
       )}
       {showCiteButton && (

vis/js/templates/listentry/PaperButtons.jsx

@@ -16,13 +16,6 @@ const CitePaperModal = ({ open, onClose, paper }) => {
 
   const citationText = paper ? getCitation(paper) : "";
 
-  const metadataMissing =
-    !paper ||
-    !paper.title ||
-    !paper.authors_objects ||
-    paper.authors_objects.length === 0 ||
-    !paper.year;
-
   return (
     // html template starts here
     <Modal id="cite_paper_modal" show={open} onHide={onClose}>
@@ -55,24 +48,6 @@ const CitePaperModal = ({ open, onClose, paper }) => {
         </div>
         <p className="cit-style-desc">{currentStyle.description}</p>
         <CopyButton textId={"copy-paper-citation"} textContent={citationText} />
-        {metadataMissing && paper && (
-          <p className="cite-paper-warning">
-            <strong>{loc.cite_metadata_warn_1}</strong>{" "}
-            {loc.cite_metadata_warn_2}{" "}
-            <a
-              href={
-                paper.list_link.isDoi
-                  ? `https://dx.doi.org/${paper.list_link.address}`
-                  : paper.list_link.address
-              }
-              target="_blank"
-              rel="noreferrer"
-            >
-              {loc.cite_metadata_warn_3}
-            </a>
-            .
-          </p>
-        )}
       </Modal.Body>
     </Modal>
     // html template ends here

vis/js/templates/modals/CitePaperModal.jsx

@@ -83,54 +83,48 @@ const getParamFilterFunction = (param, field) => {
   };
 };
 
+const SEARCHED_PROPS = [
+  "title",
+  "authors_string",
+  "published_in",
+  "year",
+  "subject_orig",
+  "tags",
+  "comments_for_filtering",
+  "resulttype",
+  "paper_abstract",
+];
+
 /**
  * Creates a paper filtering function from the search words.
  *
- * Function taken from legacy list.js
- * @param {Array} search_words array of search words (plaintext strings)
+ * @param {Array} searchedKeywords array of search keywords (plaintext strings)
  *
- * @returns {Function} filtering function
+ * @returns {Function} filtering function that returns true if paper contains all the searched keywords
  */
-const getWordFilterFunction = (search_words) => {
-  return (d) => {
-    const abstract = getPropertyOrEmptyString(d, "paper_abstract");
-    const title = getPropertyOrEmptyString(d, "title");
-    const authors = getPropertyOrEmptyString(d, "authors_string");
-    const journals = getPropertyOrEmptyString(d, "published_in");
-    const year = getPropertyOrEmptyString(d, "year");
-    const keywords = getPropertyOrEmptyString(d, "subject_orig");
-    const tags = getPropertyOrEmptyString(d, "tags");
-    const comments = getPropertyOrEmptyString(d, "comments_for_filtering");
-    const resulttype = getPropertyOrEmptyString(d, "resulttype");
-    // TODO: make these two properties language-aware
-    const open_access = d.oa ? "open access" : "";
-    const free_access = d.free_access ? "free access" : "";
-
-    let i = 0;
-    let word_found = true;
-    while (word_found && i < search_words.length) {
-      word_found =
-        abstract.indexOf(search_words[i]) !== -1 ||
-        title.indexOf(search_words[i]) !== -1 ||
-        authors.indexOf(search_words[i]) !== -1 ||
-        journals.indexOf(search_words[i]) !== -1 ||
-        year.indexOf(search_words[i]) !== -1 ||
-        keywords.indexOf(search_words[i]) !== -1 ||
-        tags.indexOf(search_words[i]) !== -1 ||
-        comments.indexOf(search_words[i]) !== -1 ||
-        resulttype.indexOf(search_words[i]) !== -1 ||
-        open_access.indexOf(search_words[i]) !== -1 ||
-        free_access.indexOf(search_words[i]) !== -1;
-      i++;
+const getWordFilterFunction = (searchedKeywords) => {
+  return (paper) => {
+    const paperKeywords = SEARCHED_PROPS.map((prop) =>
+      getPropertyOrEmptyString(paper, prop)
+    );
+
+    if (paper.oa) {
+      paperKeywords.push("open access");
+      paperKeywords.push("pdf");
     }
+    if (paper.free_access) {
+      paperKeywords.push("free access");
+    }
+
+    const paperString = paperKeywords.join(" ");
 
-    return word_found;
+    return !searchedKeywords.some((keyword) => !paperString.includes(keyword));
   };
 };
 
 const getPropertyOrEmptyString = (object, property) => {
   if (Object.prototype.hasOwnProperty.call(object, property)) {
-    return object[property].toString().toLowerCase();
+    return object[property].toString().toLowerCase().trim();
   }
 
   return "";

vis/js/utils/data.js

@@ -38,6 +38,14 @@ export const availableStyles = [
     name: "ACM",
     template: "custom-acm",
     description: 'ACM SIG Proceedings ("et al." for 3+ authors)',
+    append: (paper) => {
+      // feature request from Peter, not an official part of ACM
+      if (!paper.list_link.isDoi) {
+        return " " + paper.list_link.address;
+      }
+
+      return "";
+    },
   },
 ];
 
@@ -46,6 +54,7 @@ const useCitationStyle = () => {
 
   const getCitation = (paper) => {
     // format derived from the example JSON: https://citation.js.org/demo/
+    // full list of parameters: node_modules/@citation-js/core/lib/plugins/input/csl.js
     const cite = new Cite({
       id: paper.safe_id,
       title: paper.title,
@@ -55,17 +64,65 @@ const useCitationStyle = () => {
       })),
       issued: [{ "date-parts": paper.year.split("-") }],
       "container-title": paper.published_in,
-      DOI: paper.list_link.isDoi ? paper.list_link.address : undefined,
-      url: paper.list_link.isDoi ? undefined : paper.list_link.address,
-      // other possible attributes: "type", "volume", "issue", "page", ???
+      DOI: paper.list_link.isDoi
+        ? paper.list_link.address.replace(/(https?:\/\/)?(\w+\.)?doi.org\//, "")
+        : undefined,
+      URL: paper.list_link.isDoi ? undefined : paper.list_link.address,
+      // this information could be misleading, so we'll hide it for now:
+      //source: "Open Knowledge Maps",
+      type: getType(paper),
     });
 
-    return cite.format("bibliography", {
+    let output = cite.format("bibliography", {
       template: style.template,
     });
+
+    if (style.append) {
+      output += style.append(paper);
+    }
+
+    return output;
   };
 
   return [style, setStyle, getCitation];
 };
 
 export default useCitationStyle;
+
+// mapping of BASE type to CSL type
+const TYPE_TO_TYPE = [
+  ["Journal/newspaper article", "article-journal"],
+  ["Journal/newspaper other content", "article-journal"],
+  ["Journal/newspaper", "periodical"],
+  ["Book", "book"],
+  ["Book part", "chapter"],
+  ["Conference object", "paper-conference"],
+  ["Dataset", "dataset"],
+  ["Manuscript", "manuscript"],
+  ["Map", "map"],
+  ["Moving image/video", "motion_picture"],
+  ["Patent", "patent"],
+  ["Report", "report"],
+  ["Review", "review"],
+  ["Software", "software"],
+  ["Still image", "graphic"],
+  ["Thesis", "thesis"],
+  ["Thesis: bachelor", "thesis"],
+  ["Thesis: doctoral and postdoctoral", "thesis"],
+  ["Thesis: master", "thesis"],
+];
+
+const getType = (paper) => {
+  const types = paper.resulttype;
+  if (!types || types.length === 0) {
+    return undefined;
+  }
+
+  for (const entry of TYPE_TO_TYPE) {
+    if (types.includes(entry[0])) {
+      return entry[1];
+    }
+  }
+
+  return undefined;
+};