adding images to master set folder along with fixing script

2026-05-08 23:04:21 -07:00
parent b39fc6e7db
commit 37d1d5fe4e
3699 changed files with 9418 additions and 33 deletions
@@ -1,34 +1,72 @@
-const fs = require('fs');
-const path = require('path');
-const csvParser = require('csv-parser');
+import pLimit from 'p-limit';
+import fs from 'fs';
+import path from 'path';
+import csvParser from 'csv-parser';

-// Define the target directory for saving images
-let imageDir = './master-set-images/perfect-order';
-if (!fs.existsSync(imageDir)) {
-    fs.mkdirSync(imageDir);
+const limit = pLimit(10);
+
+// Define the target data structures for each set of images
+let sets = [
+    { dir: './master-set-images/perfect-order', csv: './csv/ME03PerfectOrderProductsAndPrices.csv' },
+    { dir: './master-set-images/mega-evolutions', csv: './csv/ME01MegaEvolutionProductsAndPrices.csv' },
+    { dir: './master-set-images/phantasmal-flames', csv: './csv/ME02PhantasmalFlamesProductsAndPrices.csv' },
+    { dir: './master-set-images/chaos-rising', csv: './csv/ME04ChaosRisingProductsAndPrices.csv' },
+    { dir: './master-set-images/ascended-heroes', csv: './csv/MEAscendedHeroesProductsAndPrices.csv' },
+    { dir: './master-set-images/obsidian-flames', csv: './csv/SV03ObsidianFlamesProductsAndPrices.csv' },
+    { dir: './master-set-images/twilight-masquerade', csv: './csv/SV06TwilightMasqueradeProductsAndPrices.csv' },
+    { dir: './master-set-images/surging-sparks', csv: './csv/SV08SurgingSparksProductsAndPrices.csv' },
+    { dir: './master-set-images/journey-together', csv: './csv/SV09JourneyTogetherProductsAndPrices.csv' },
+    { dir: './master-set-images/destined-rivals', csv: './csv/SV10DestinedRivalsProductsAndPrices.csv' },
+    { dir: './master-set-images/black-bolt', csv: './csv/SVBlackBoltProductsAndPrices.csv' },
+    { dir: './master-set-images/prismatic-evolutions', csv: './csv/SVPrismaticEvolutionsProductsAndPrices.csv' },
+    { dir: './master-set-images/scarlet-and-violet-151', csv: './csv/SVScarletAndViolet151ProductsAndPrices.csv' },
+    { dir: './master-set-images/white-flare', csv: './csv/SVWhiteFlareProductsAndPrices.csv' },
+    // Add more sets as needed
+];
+
+// Check if each directory exists, create them if not
+sets.forEach(set => {
+    const dir = set.dir;
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir);
+    }
+});
+
+async function main() {
+    for (const set of sets) {
+        // Read the CSV file using csv-parser
+        const stream = fs.createReadStream(set.csv)
+            .pipe(csvParser());
+        let products = [];
+        const fetchPromises = [];
+
+        stream.on('data', row => products.push(row));
+        stream.on('end', () => {
+            // Loop through each product and download its image if the file doesn't exist
+            products.forEach(product => {
+                const imgUrl = product['imageUrl'];
+                const fileName = product['productId'] + '.jpg';
+                const filePath = path.join(set.dir, fileName);
+                fs.access(filePath, (err) => {
+                    if (err) { // File doesn't exist
+                        fetchPromises.push(
+                            limit(() => {
+                                console.log('Downloading', imgUrl);
+                                return fetch(imgUrl)
+                                    .then(res => res.arrayBuffer())
+                                    .then(buffer => fs.writeFileSync(filePath, Buffer.from(buffer)))
+                                    .catch(err => console.error(err));
+                            })
+                        );
+                    } else { // File exists
+                        console.log('Skipping', fileName);
+                    }
+                });
+            });
+        });
+
+        await Promise.all(fetchPromises);
+    }
 }

-// Read the CSV file using csv-parser
-const stream = fs.createReadStream('./csv/ME03PerfectOrderProductsAndPrices.csv')
-    .pipe(csvParser());
-let products = [];
-stream.on('data', row => products.push(row));
-stream.on('end', () => {
-    // Loop through each product and download its image if the file doesn't exist
-    products.forEach(product => {
-        const imgUrl = product['imageUrl'];
-        const fileName = product['productId'] + '.jpg';
-        const filePath = path.join(imageDir, fileName);
-        fs.access(filePath, (err) => {
-            if (err) { // File doesn't exist
-                console.log('Downloading', imgUrl);
-                fetch(imgUrl)
-                    .then(res => res.arrayBuffer())
-                    .then(buffer => fs.writeFileSync(filePath, Buffer.from(buffer)))
-                    .catch(err => console.error(err));
-            } else { // File exists
-                console.log('Skipping', fileName);
-            }
-        });
-    });
-});
+main();