fix(scraper): ignore tests on scrape

Author: jonasstenberg

apps/functions/notify-slack/package.json

@@ -13,20 +13,20 @@
   "license": "MIT",
   "dependencies": {
     "@google-cloud/functions-framework": "3.2.0",
-    "@google-cloud/storage": "^5.19.3",
-    "@types/express": "^4.17.13",
+    "@google-cloud/storage": "^5.20.5",
+    "@types/express": "^4.17.20",
     "dotenv": "16.0.3",
     "form-data": "^4.0.0",
-    "node-fetch": "^2.6.7",
-    "zod": "3.20.2"
+    "node-fetch": "^2.7.0",
+    "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@devolunch/shared": "workspace:",
-    "@pnpm/make-dedicated-lockfile": "^0.5.5",
+    "@devolunch/shared": "workspace:^",
+    "@pnpm/make-dedicated-lockfile": "^0.5.10",
     "@types/node": "20.1.7",
-    "@types/node-fetch": "^2.6.1",
-    "eslint-config-custom": "workspace:",
+    "@types/node-fetch": "^2.6.7",
+    "eslint-config-custom": "workspace:^",
     "prettier": "2.8.8",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   }
 }

apps/functions/notify-slack/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "ESNext",
     "module": "NodeNext",
-    "moduleResolution": "node",
+    "moduleResolution": "NodeNext",
     "lib": ["ESNext", "dom", "dom.iterable"],
     "esModuleInterop": true,
     "strict": true,

apps/functions/scraper/jest.config.cjs

@@ -5,7 +5,13 @@ module.exports = {
   moduleNameMapper: {
     '^@/(.*)$': '<rootDir>/src/$1',
   },
+  testMatch: ['**/src/**/*.test.(ts|tsx)'],
   transform: {
-    '^.+\\.tsx?$': ['ts-jest', {}],
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        tsconfig: 'tsconfig.jest.json',
+      },
+    ],
   },
 };

apps/functions/scraper/package.json

@@ -17,25 +17,25 @@
   "license": "MIT",
   "dependencies": {
     "@google-cloud/functions-framework": "3.2.0",
-    "@google-cloud/storage": "^5.19.3",
+    "@google-cloud/storage": "^5.20.5",
     "@google-cloud/translate": "^6.3.1",
     "dotenv": "16.0.3",
     "pdf-parse": "1.1.1",
-    "puppeteer": "^20.1.2",
+    "puppeteer": "^20.9.0",
     "sharp": "0.32.1",
-    "zod": "3.20.2"
+    "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@devolunch/shared": "workspace:",
-    "@pnpm/make-dedicated-lockfile": "^0.5.5",
-    "@types/express": "^4.17.13",
-    "@types/jest": "^29.5.5",
+    "@devolunch/shared": "workspace:^",
+    "@pnpm/make-dedicated-lockfile": "^0.5.10",
+    "@types/express": "^4.17.20",
+    "@types/jest": "^29.5.6",
     "@types/node": "20.1.7",
     "@types/pdf-parse": "1.1.1",
-    "eslint-config-custom": "workspace:",
+    "eslint-config-custom": "workspace:^",
     "jest": "^29.7.0",
     "prettier": "2.8.8",
     "ts-jest": "^29.1.1",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   }
 }

apps/functions/scraper/src/scraper.ts

@@ -13,7 +13,9 @@ const TIMEOUT = 120000;
 export const getRestaurantFilePaths = async (dir: string) => {
   const files = await fs.readdir(dir);
   let targetFiles = files.filter((file) => {
-    return path.extname(file).toLowerCase() === '.js';
+    const extname = path.extname(file).toLowerCase();
+    const isTestFile = file.toLowerCase().endsWith('.test.js');
+    return extname === '.js' && !isTestFile;
   });
 
   const filesOverride = config.filesOverride?.split(',');
@@ -39,6 +41,7 @@ export const scrapeRestaurant = async (browser: Browser, dir: string, file: stri
     const dishes = await browserScrapeFunction(page);
 
     console.log(`Resizing image for ${restaurantMeta.title}`);
+
     // upload image to bucket if there are any
     const imageUrl = await resizeImage(restaurantMeta.imageUrl, restaurantMeta.title, {
       size: {

apps/functions/scraper/tsconfig.jest.json

@@ -1,7 +1,7 @@
 {
   "extends": "./tsconfig.json",
   "compilerOptions": {
-    "esModuleInterop": true
+    "module": "NodeNext"
   },
   "include": ["src/**/*.test.ts"]
 }

apps/functions/scraper/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "ESNext",
     "module": "ESNext",
-    "moduleResolution": "node",
+    "moduleResolution": "NodeNext",
     "lib": ["ESNext", "dom", "dom.iterable"],
     "esModuleInterop": true,
     "strict": true,

apps/server/package.json

@@ -11,27 +11,27 @@
   },
   "dependencies": {
     "@devolunch/shared": "workspace:*",
-    "@google-cloud/storage": "^5.19.3",
-    "@slack/web-api": "^6.7.1",
+    "@google-cloud/storage": "^5.20.5",
+    "@slack/web-api": "^6.9.0",
     "compression": "1.7.4",
     "cors": "^2.8.5",
     "dotenv": "16.0.3",
-    "express": "^4.17.3",
-    "zod": "3.20.2"
+    "express": "^4.18.2",
+    "zod": "^3.22.4"
   },
   "devDependencies": {
-    "eslint-config-custom": "workspace:*",
-    "@tsconfig/node18": "^2.0.0",
+    "@tsconfig/node18": "^2.0.1",
     "@types/compression": "1.7.2",
-    "@types/cors": "^2.8.12",
-    "@types/express": "^4.17.13",
-    "@types/node": "^20.1.0",
-    "@types/node-fetch": "^2.6.1",
-    "nodemon": "^2.0.15",
-    "pino": "^7.10.0",
+    "@types/cors": "^2.8.15",
+    "@types/express": "^4.17.20",
+    "@types/node": "^20.8.7",
+    "@types/node-fetch": "^2.6.7",
+    "eslint-config-custom": "workspace:*",
+    "nodemon": "^2.0.22",
+    "pino": "^7.11.0",
     "pino-pretty": "^7.6.1",
     "prettier": "2.6.2",
-    "ts-node": "^10.7.0",
-    "typescript": "^5.0.4"
+    "ts-node": "^10.9.1",
+    "typescript": "^5.2.2"
   }
 }

apps/server/tsconfig.json

@@ -13,7 +13,7 @@
     "noUnusedLocals": true,
     "skipLibCheck": true,
     "sourceMap": true,
-    "moduleResolution": "node",
+    "moduleResolution": "NodeNext",
     "composite": false,
     "baseUrl": "."
   },

package.json

@@ -20,8 +20,20 @@
     "preinstall": "npx only-allow pnpm"
   },
   "devDependencies": {
-    "husky": "^8.0.0",
+    "husky": "^8.0.3",
     "prettier": "^2.8.8",
     "turbo": "1.10.7"
+  },
+  "pnpm": {
+    "overrides": {
+      "got@<11.8.5": ">=11.8.5",
+      "protobufjs@>=6.10.0 <6.11.4": ">=6.11.4",
+      "vite@>=4.3.0 <4.3.9": ">=4.3.9",
+      "semver@<5.7.2": ">=5.7.2",
+      "semver@>=6.0.0 <6.3.1": ">=6.3.1",
+      "get-func-name@<2.0.1": ">=2.0.1",
+      "zod@<=3.22.2": ">=3.22.3",
+      "postcss@<8.4.31": ">=8.4.31"
+    }
   }
 }