diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 16f4f95..50cdb65 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -15,44 +15,10 @@ jobs: registry: gitea.nelson-household.com # Replace with your Gitea domain username: ${{ gitea.actor }} password: ${{ secrets.RUNNER_TOKEN }} - - name: Delete Old Images and Containers (Docker versions < 1.10) + - name: Delete Old Images and Containers run: | - CONTAINERS=$(docker ps -aq '--filter=reference='$(echo gitea\.nelson-household\.com\/joseph\.nelson4456\/koillection-uploader\/koilcollection-uploader | grep -oE '[^/]+' | head -1) || true) - IMAGES=$(docker images -q '--filter=reference='$(echo gitea\.nelson-household\.com\/joseph\.nelson4456\/koillection-uploader\/koilcollection-uploader | grep -oE '[^/]+' | head -1) || true) - if [ ! -z "$CONTAINERS" ]; then - echo "Deleting Containers:" - docker rm $CONTAINERS - fi - if [ ! -z "$IMAGES" ]; then - echo "Deleting Images:" - docker rmi $IMAGES - fi - - name: Delete Old Images and Containers (Docker version >= 1.10) - if: | docker version --format '{{ .Server.Version }}' | grep -q '1.10' || true; then - run: | - CONTAINERS=$(docker ps -aq --filter reference=gitea\.nelson-household\.com\/joseph\.nelson4456\/koillection-uploader\/koilcollection-uploader || true) - IMAGES=$(docker images -q --filter reference=gitea\.nelson-household\.com\/joseph\.nelson4456\/koillection-uploader\/koilcollection-uploader || true) - if [ ! -z "$CONTAINERS" ]; then - echo "Deleting Containers:" - docker rm $CONTAINERS - fi - if [ ! -z "$IMAGES" ]; then - echo "Deleting Images:" - docker rmi $IMAGES - fi - else - run: | - CONTAINERS=$(docker ps -aq '--filter=reference='$(echo gitea\.nelson-household\.com\/joseph\.nelson4456\/koillection-uploader\/koilcollection-uploader | grep -oE '[^/]+' | head -1) || true) - IMAGES=$(docker images -q '--filter=reference='$(echo gitea\.nelson-household\.com\/joseph\.nelson4456\/koillection-uploader\/koilcollection-uploader | grep -oE '[^/]+' | head -1) || true) - if [ ! -z "$CONTAINERS" ]; then - echo "Deleting Containers:" - docker rm $CONTAINERS - fi - if [ ! -z "$IMAGES" ]; then - echo "Deleting Images:" - docker rmi $IMAGES - fi - endif + docker rm $(docker ps -a -q -f "name=koillection-uploader") + docker rmi $(docker images --format "{{.Repository}}:{{.Tag}}" | grep "koillection-uploader") - name: Build and Push Image run: | docker build -t gitea.nelson-household.com/joseph.nelson4456/koillection-uploader/koilcollection-uploader:${{ github.sha }} . diff --git a/image-downloader.js b/image-downloader.js index 13ea896..c1caf58 100644 --- a/image-downloader.js +++ b/image-downloader.js @@ -1,4 +1,6 @@ const fs = require('fs'); +const path = require('path'); +const csvParser = require('csv-parser'); // Define the target directory for saving images let imageDir = './master-set-images/perfect-order'; @@ -6,25 +8,27 @@ if (!fs.existsSync(imageDir)) { fs.mkdirSync(imageDir); } -// Read the CSV file line by line -const stream = fs.createReadStream('./csv/ME03PerfectOrderProductsAndPrices.csv'); -let data = []; -stream.on('data', chunk => data.push(chunk)); +// Read the CSV file using csv-parser +const stream = fs.createReadStream('./csv/ME03PerfectOrderProductsAndPrices.csv') + .pipe(csvParser()); +let products = []; +stream.on('data', row => products.push(row)); stream.on('end', () => { - let rows = data.toString().split('\n').slice(1); - - // Loop through each row (product) and download its image - rows.forEach(row => { - const columns = row.split(/[\t,]/).map(c => c.trim()); - - if (!!columns[3]) { // Image URL exists - let imgUrl = columns[3]; // Change image resolution - const fileName = columns[0]; // Get the file name (cleanName) - console.log(imgUrl); - fetch(imgUrl) - .then(res => res.arrayBuffer()) - .then(buffer => fs.writeFileSync(`${imageDir}/${fileName}.jpg`, Buffer.from(buffer))) - .catch(err => console.error(err)); - } + // Loop through each product and download its image if the file doesn't exist + products.forEach(product => { + const imgUrl = product['imageUrl']; + const fileName = product['productId'] + '.jpg'; + const filePath = path.join(imageDir, fileName); + fs.access(filePath, (err) => { + if (err) { // File doesn't exist + console.log('Downloading', imgUrl); + fetch(imgUrl) + .then(res => res.arrayBuffer()) + .then(buffer => fs.writeFileSync(filePath, Buffer.from(buffer))) + .catch(err => console.error(err)); + } else { // File exists + console.log('Skipping', fileName); + } + }); }); });