From 77d1f568141226943b2aae9a7b1b7d45bbcd7416 Mon Sep 17 00:00:00 2001 From: Joseph Nelson Date: Tue, 28 Apr 2026 22:41:28 -0700 Subject: [PATCH] pushing latest code changes --- createCollectionsFromCSVs.js | 166 +++++++++++++++++++++++------------ 1 file changed, 111 insertions(+), 55 deletions(-) diff --git a/createCollectionsFromCSVs.js b/createCollectionsFromCSVs.js index b8a517e..b4019ac 100644 --- a/createCollectionsFromCSVs.js +++ b/createCollectionsFromCSVs.js @@ -74,53 +74,48 @@ const createItem = async (payload, token) => { throw error; } }; -async function downloadTcgImage(itemId, imageUrl) { +async function downloadTcgImage(productId, imageUrl) { try { - // 1. Fetch the image data using fetch - const response = await fetch(imageUrl, { - method: 'GET', // Use GET for image retrieval - responseType: 'stream' // Important for streaming the image - }); - - if (!response.ok) { - throw new Error(`Failed to fetch image: ${response.statusText}`); + // 1. Ensure directory exists + if (!fs.existsSync(imagesDir)) { + fs.mkdirSync(imagesDir, { recursive: true }); } - - // 2. Download the image using your existing logic - const dest = path.join(imagesDir, `${itemId}_${imageUrl.split('/').pop()}`); - await new Promise((resolve, reject) => { - response.body.pipe(fs.createWriteStream(dest)); - response.body.on('error', reject); - (async () => { - try { - await new Promise((resolve, reject) => { - response.body.on('finish', () => resolve()); - }); - } catch (error) { - reject(error); - } - })(); - }); - - console.log(`Image saved to ${dest}`); + // 2. Construct file destination + const filename = `${productId}_${path.basename(new URL(imageUrl).pathname)}`; + const dest = path.join(imagesDir, filename); + // 3. Fetch the image + const response = await fetch(imageUrl); + if (!response.ok) { + console.error(`Failed to fetch ${imageUrl}: ${response.statusText}`); + return {}; + } + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + fs.writeFileSync(dest, buffer); // Use the correct destination path + console.log(`Successfully saved: ${dest}`); return { success: true, destination: dest }; } catch (error) { - console.error('Error applying image:', error); - throw error; + console.error('Error downloading image:', error); + return {}; } } const applyImageToItem = async (itemId, localImagePath, token) => { try { + console.log(`Local Image Path: ${localImagePath}`); + + if (!fs.existsSync(localImagePath)) { + console.log('Looks like the image does not exists?'); + return {}; + } + + const formData = new FormData(); + formData.append(localImagePath, fs.readFileSync(localImagePath)); const response = await fetch(`${koillectionBaseUrl}/items/${itemId}/image`, { method: 'POST', headers: { 'Authorization': `Bearer ${token}`, - 'Content-Type': 'multipart/form-data', // Important for file uploads }, - body: new FormData({ - append: 'image', - value: new Blob([fs.readFileSync(localImagePath)]), // Use fs.readFileSync to read the local file - }, 'image.jpg') + body: formData }); if (!response.ok) { @@ -138,47 +133,108 @@ const applyImageToItem = async (itemId, localImagePath, token) => { const parseCsv = async (collectionId, csvFilePath, token) => { try { const rows = []; - fs.createReadStream(csvFilePath) - .pipe(csvParser()) - .on('data', (row) => rows.push(row)) - .on('end', async () => { - rows.map(async (row) => { - //console.log(`Here is the row wer are on: ${JSON.stringify(row, null, 2)}`); - const payload = { - name: row.name, - collection: `/api/collections/${collectionId}`, - visibility: 'public', - }; + const results = {}; + const stream = fs.createReadStream(csvFilePath) + .pipe(csvParser()); // Now using the stream directly - try { + return await new Promise((resolve, reject) => { + stream.on('data', (row) => rows.push(row)); + stream.on('end', async () => { + try { + const processingPromises = rows.map(async (row) => { + const payload = { + name: row.name, + collection: `/api/collections/${collectionId}`, + visibility: 'public', + }; const createdItem = await createItem(payload, token); console.log(`Created item: ${createdItem.name}`); - const imageObj = await downloadTcgImage(createdItem.id, row.imageUrl); - const image = await applyImageToItem(createdItem.id, imageObj.destination, token); - console.log(`Applied image to item: ${createdItem.name}`); - } catch (itemError) { - console.error(`Error creating item for row ${row.name}:`, itemError); - } - }); + //const imageUrl = await downloadTcgImage(itemId, row.imageUrl); // Await the image download + results[row.productId] = { imageUrl: row.imageUrl, itemId: createdItem.id }; + }); + + await Promise.all(processingPromises); + + resolve(results); + } catch (err) { + reject(err); + } }); + + stream.on('error', (err) => reject(err)); + }); } catch (error) { console.error(`Error creating collection ${name} from CSV:`, error); + return {}; } } +function mergeObjects(o1, o2) { + const merged = { ...o1 }; + + for (const key in o2) { + if (merged[key]) { + // Key exists, convert itemId to itemIds array + const existing = merged[key]; + const incoming = o2[key]; + + // Keep existing itemId if it's already an array, otherwise make it one + const items = Array.isArray(existing.itemIds) ? existing.itemIds : [existing.itemId]; + items.push(incoming.itemId); + + merged[key] = { + ...existing, + ...incoming, + itemIds: items, + itemId: undefined // Remove old field + }; + // Optionally cleanup undefined + delete merged[key].itemId; + } else { + // Key is new, add as is + merged[key] = o2[key]; + } + } + return merged; +} const createCollectionFromCSV = async (name, csvFilePath) => { const token = await createJWTToken('bigassdragon', 'd7Bis4Y4EUhfMf'); const joeCollectionId = await createCollectionIfNotExists(name, '019dc310-a130-799e-8c0d-db90774b7b77', token); const tylerCollectionId = await createCollectionIfNotExists(name, '019dc310-d175-7e59-b465-c8e684ee7186', token); - await parseCsv(joeCollectionId, csvFilePath, token); - await parseCsv(tylerCollectionId, csvFilePath, token); + const joeCollectionData = await parseCsv(joeCollectionId, csvFilePath, token).then((results) => results); + const tylerCollectionData = await parseCsv(tylerCollectionId, csvFilePath, token).then((results) => results); + const mergedCollectionData = mergeObjects(joeCollectionData, tylerCollectionData); + const itemIdImageLocations = []; + + for (const productId of Object.keys(mergedCollectionData)) { + const item = mergedCollectionData[productId]; + // Await each download before starting the next + const { destination } = await downloadTcgImage(productId, item.imageUrl); + + item.itemIds.forEach((id) => itemIdImageLocations.push({ itemId: id, imageDest: destination })); + } + + // After processing all collections, check the imagesDir + console.log("Finished processing collections."); + const files = fs.readdirSync(imagesDir); + console.log(`Files in ${imagesDir}: ${files}`); + console.log(`Item ID Image Locations: ${JSON.stringify(itemIdImageLocations, null, 2)}`); + // Process multiple images concurrently using Promise.all() + const promises = itemIdImageLocations.map(async (item) => { + await applyImageToItem(item.itemId, item.imageDest, token); + }); + await Promise.all(promises); }; // Main execution async function main() { for (const collectionName in collectionsMapping) { try { + console.log(`Processing collection: ${collectionName}`); await createCollectionFromCSV(collectionName, collectionsMapping[collectionName]); + console.log(`Collection ${collectionName} processed successfully.`); } catch (error) { console.error(`Error processing collection - ${collectionName}:`, error); + // Break the loop if an error occurs + break; } } }