73 lines
3.3 KiB
JavaScript
73 lines
3.3 KiB
JavaScript
import pLimit from 'p-limit';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import csvParser from 'csv-parser';
|
|
|
|
const limit = pLimit(10);
|
|
|
|
// Define the target data structures for each set of images
|
|
let sets = [
|
|
{ dir: './master-set-images/perfect-order', csv: './csv/ME03PerfectOrderProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/mega-evolutions', csv: './csv/ME01MegaEvolutionProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/phantasmal-flames', csv: './csv/ME02PhantasmalFlamesProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/chaos-rising', csv: './csv/ME04ChaosRisingProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/ascended-heroes', csv: './csv/MEAscendedHeroesProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/obsidian-flames', csv: './csv/SV03ObsidianFlamesProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/twilight-masquerade', csv: './csv/SV06TwilightMasqueradeProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/surging-sparks', csv: './csv/SV08SurgingSparksProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/journey-together', csv: './csv/SV09JourneyTogetherProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/destined-rivals', csv: './csv/SV10DestinedRivalsProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/black-bolt', csv: './csv/SVBlackBoltProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/prismatic-evolutions', csv: './csv/SVPrismaticEvolutionsProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/scarlet-and-violet-151', csv: './csv/SVScarletAndViolet151ProductsAndPrices.csv' },
|
|
{ dir: './master-set-images/white-flare', csv: './csv/SVWhiteFlareProductsAndPrices.csv' },
|
|
// Add more sets as needed
|
|
];
|
|
|
|
// Check if each directory exists, create them if not
|
|
sets.forEach(set => {
|
|
const dir = set.dir;
|
|
if (!fs.existsSync(dir)) {
|
|
fs.mkdirSync(dir);
|
|
}
|
|
});
|
|
|
|
async function main() {
|
|
for (const set of sets) {
|
|
// Read the CSV file using csv-parser
|
|
const stream = fs.createReadStream(set.csv)
|
|
.pipe(csvParser());
|
|
let products = [];
|
|
const fetchPromises = [];
|
|
|
|
stream.on('data', row => products.push(row));
|
|
stream.on('end', () => {
|
|
// Loop through each product and download its image if the file doesn't exist
|
|
products.forEach(product => {
|
|
const imgUrl = product['imageUrl'];
|
|
const fileName = product['productId'] + '.jpg';
|
|
const filePath = path.join(set.dir, fileName);
|
|
fs.access(filePath, (err) => {
|
|
if (err) { // File doesn't exist
|
|
fetchPromises.push(
|
|
limit(() => {
|
|
console.log('Downloading', imgUrl);
|
|
return fetch(imgUrl)
|
|
.then(res => res.arrayBuffer())
|
|
.then(buffer => fs.writeFileSync(filePath, Buffer.from(buffer)))
|
|
.catch(err => console.error(err));
|
|
})
|
|
);
|
|
} else { // File exists
|
|
console.log('Skipping', fileName);
|
|
}
|
|
});
|
|
});
|
|
});
|
|
|
|
await Promise.all(fetchPromises);
|
|
}
|
|
}
|
|
|
|
main();
|