From 7bd4f3de0e3da1b1a26b5ab6fd392e5b7732e54d Mon Sep 17 00:00:00 2001 From: 2ManyProjects Date: Mon, 5 May 2025 17:48:46 -0500 Subject: [PATCH] Batched bulk uploads --- backend/src/index.js | 3 +- db/init/02-seed.sql | 329 +++++++++++----------- frontend/src/pages/Admin/ProductsPage.jsx | 75 ++++- 3 files changed, 233 insertions(+), 174 deletions(-) diff --git a/backend/src/index.js b/backend/src/index.js index d42a89a..ad7a5c4 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -157,7 +157,8 @@ app.use(cors({ })); app.use('/api/payment', stripePaymentRoutes(pool, query, authMiddleware(pool, query))); -app.use(express.json()); +app.use(express.json({ limit: '10mb' })); +app.use(express.urlencoded({ extended: true, limit: '10mb' })); app.use(morgan('dev')); // Serve static files - serve the entire public directory diff --git a/db/init/02-seed.sql b/db/init/02-seed.sql index a41f72b..c2cbcca 100644 --- a/db/init/02-seed.sql +++ b/db/init/02-seed.sql @@ -4,183 +4,182 @@ -- Note: Product categories are already inserted in the schema file, so we're skipping that step -- Insert rock products -WITH categories AS ( - SELECT id, name FROM product_categories -) --- Insert rock products -INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) -SELECT - 'Amethyst Geode', - 'Beautiful purple amethyst geode with crystal formation', - id, - 49.99, - 15, - 450.0, - 'Purple', - 'Quartz', - 'Brazil', - NULL -FROM categories WHERE name = 'Rock'; +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) +-- SELECT +-- 'Amethyst Geode', +-- 'Beautiful purple amethyst geode with crystal formation', +-- id, +-- 49.99, +-- 15, +-- 450.0, +-- 'Purple', +-- 'Quartz', +-- 'Brazil', +-- NULL +-- FROM categories WHERE name = 'Rock'; -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) -SELECT - 'Polished Labradorite', - 'Stunning polished labradorite with iridescent blue flash', - id, - 29.99, - 25, - 120.5, - 'Gray/Blue', - 'Feldspar', - 'Madagascar', - NULL -FROM categories WHERE name = 'Rock'; +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) +-- SELECT +-- 'Polished Labradorite', +-- 'Stunning polished labradorite with iridescent blue flash', +-- id, +-- 29.99, +-- 25, +-- 120.5, +-- 'Gray/Blue', +-- 'Feldspar', +-- 'Madagascar', +-- NULL +-- FROM categories WHERE name = 'Rock'; -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) -SELECT - 'Raw Turquoise', - 'Natural turquoise specimen, unpolished', - id, - 19.99, - 18, - 85.2, - 'Turquoise', - 'Turquoise', - 'Arizona', - NULL -FROM categories WHERE name = 'Rock'; +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) +-- SELECT +-- 'Raw Turquoise', +-- 'Natural turquoise specimen, unpolished', +-- id, +-- 19.99, +-- 18, +-- 85.2, +-- 'Turquoise', +-- 'Turquoise', +-- 'Arizona', +-- NULL +-- FROM categories WHERE name = 'Rock'; --- Insert bone products -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url) -SELECT - 'Deer Antler', - 'Naturally shed deer antler, perfect for display or crafts', - id, - 24.99, - 8, - 38.5, - 'Antler', - NULL -FROM categories WHERE name = 'Bone'; +-- -- Insert bone products +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url) +-- SELECT +-- 'Deer Antler', +-- 'Naturally shed deer antler, perfect for display or crafts', +-- id, +-- 24.99, +-- 8, +-- 38.5, +-- 'Antler', +-- NULL +-- FROM categories WHERE name = 'Bone'; -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url) -SELECT - 'Fossil Fish', - 'Well-preserved fossil fish from the Green River Formation', - id, - 89.99, - 5, - 22.8, - 'Fossilized Bone', - NULL -FROM categories WHERE name = 'Bone'; +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url) +-- SELECT +-- 'Fossil Fish', +-- 'Well-preserved fossil fish from the Green River Formation', +-- id, +-- 89.99, +-- 5, +-- 22.8, +-- 'Fossilized Bone', +-- NULL +-- FROM categories WHERE name = 'Bone'; --- Insert stick products -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) -SELECT - 'Driftwood Piece', - 'Unique driftwood piece, weathered by the ocean', - id, - 14.99, - 12, - 45.6, - 8.3, - 'Driftwood', - 'Tan/Gray', - NULL -FROM categories WHERE name = 'Stick'; +-- -- Insert stick products +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) +-- SELECT +-- 'Driftwood Piece', +-- 'Unique driftwood piece, weathered by the ocean', +-- id, +-- 14.99, +-- 12, +-- 45.6, +-- 8.3, +-- 'Driftwood', +-- 'Tan/Gray', +-- NULL +-- FROM categories WHERE name = 'Stick'; -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) -SELECT - 'Walking Stick', - 'Hand-selected natural maple walking stick', - id, - 34.99, - 10, - 152.4, - 3.8, - 'Maple', - 'Brown', - NULL -FROM categories WHERE name = 'Stick'; +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) +-- SELECT +-- 'Walking Stick', +-- 'Hand-selected natural maple walking stick', +-- id, +-- 34.99, +-- 10, +-- 152.4, +-- 3.8, +-- 'Maple', +-- 'Brown', +-- NULL +-- FROM categories WHERE name = 'Stick'; -WITH categories AS ( - SELECT id, name FROM product_categories -) -INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) -SELECT - 'Decorative Branch Set', - 'Set of 3 decorative birch branches for home decoration', - id, - 19.99, - 20, - 76.2, - 1.5, - 'Birch', - 'White', - NULL -FROM categories WHERE name = 'Stick'; +-- WITH categories AS ( +-- SELECT id, name FROM product_categories +-- ) +-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) +-- SELECT +-- 'Decorative Branch Set', +-- 'Set of 3 decorative birch branches for home decoration', +-- id, +-- 19.99, +-- 20, +-- 76.2, +-- 1.5, +-- 'Birch', +-- 'White', +-- NULL +-- FROM categories WHERE name = 'Stick'; --- Create a cart for testing -INSERT INTO carts (user_id) -SELECT id FROM users WHERE email = 'jane@example.com'; +-- -- Create a cart for testing +-- INSERT INTO carts (user_id) +-- SELECT id FROM users WHERE email = 'jane@example.com'; --- Add product tags - using a different approach --- Tag: Decorative for Amethyst Geode -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative'; +-- -- Add product tags - using a different approach +-- -- Tag: Decorative for Amethyst Geode +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative'; --- Tag: Polished for Polished Labradorite -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished'; +-- -- Tag: Polished for Polished Labradorite +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished'; --- Tag: Raw for Raw Turquoise -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw'; +-- -- Tag: Raw for Raw Turquoise +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw'; --- Tags: Fossil and Educational for Fossil Fish -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil'; +-- -- Tags: Fossil and Educational for Fossil Fish +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil'; -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Fossil Fish' AND t.name = 'Educational'; +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Fossil Fish' AND t.name = 'Educational'; --- Tag: Decorative for Driftwood Piece -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative'; +-- -- Tag: Decorative for Driftwood Piece +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative'; --- Tag: Collectible for Walking Stick -INSERT INTO product_tags (product_id, tag_id) -SELECT p.id, t.id -FROM products p, tags t -WHERE p.name = 'Walking Stick' AND t.name = 'Collectible'; \ No newline at end of file +-- -- Tag: Collectible for Walking Stick +-- INSERT INTO product_tags (product_id, tag_id) +-- SELECT p.id, t.id +-- FROM products p, tags t +-- WHERE p.name = 'Walking Stick' AND t.name = 'Collectible'; \ No newline at end of file diff --git a/frontend/src/pages/Admin/ProductsPage.jsx b/frontend/src/pages/Admin/ProductsPage.jsx index 2ca54eb..a210389 100644 --- a/frontend/src/pages/Admin/ProductsPage.jsx +++ b/frontend/src/pages/Admin/ProductsPage.jsx @@ -23,7 +23,8 @@ import { DialogContentText, DialogTitle, Stack, - Tooltip + Tooltip, + LinearProgress } from '@mui/material'; import { Edit as EditIcon, @@ -468,7 +469,7 @@ const AdminProductsPage = () => { }; // Submit bulk upload - const handleSubmitUpload = () => { + const handleSubmitUpload = async () => { if (parsedProducts.length === 0) { setUploadError('No valid products found in the file. Please check your data.'); return; @@ -477,19 +478,64 @@ const AdminProductsPage = () => { setIsUploading(true); setUploadError(null); - // Submit products in batches of 20 to respect connection limits - const batchSize = 20; + // Process products in batches to avoid payload size limits + const batchSize = 50; // Smaller batch size to avoid payload limits const batches = []; for (let i = 0; i < parsedProducts.length; i += batchSize) { - const batch = parsedProducts.slice(i, i + batchSize); - batches.push(batch); + batches.push(parsedProducts.slice(i, i + batchSize)); } - // Process first batch - bulkUploadProducts.mutate(parsedProducts); + try { + // Track successes and errors across all batches + let allSuccesses = []; + let allErrors = []; + + // Process batches sequentially to avoid overwhelming the server + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + setUploadProgress(Math.round((i / batches.length) * 100)); + + try { + // Send this batch to the server + const response = await apiClient.post('/admin/products', { products: batch }); + + // Accumulate successes and errors + if (response.data.success) allSuccesses = [...allSuccesses, ...response.data.success]; + if (response.data.errs) allErrors = [...allErrors, ...response.data.errs]; + + } catch (error) { + console.error(`Error processing batch ${i+1}:`, error); + // Add error for each product in the failed batch + const batchErrors = batch.map(product => ({ + error: true, + message: `Server error: ${error.message || 'Unknown error'}`, + code: error.response?.status || 500, + product: { name: product.name } + })); + allErrors = [...allErrors, ...batchErrors]; + } + } + + // All batches processed - invalidate product query to refresh the list + queryClient.invalidateQueries({ queryKey: ['admin-products'] }); + + // Show results to user + setUploadResults({ + success: allSuccesses, + errs: allErrors + }); + + setUploadSuccess(true); + } catch (error) { + setUploadError(`Failed to upload products: ${error.message}`); + } finally { + setIsUploading(false); + setUploadProgress(0); + } }; + // Filter and paginate products const filteredProducts = products || []; const paginatedProducts = filteredProducts.slice( @@ -819,6 +865,19 @@ const AdminProductsPage = () => { )} + {/* Show progress bar when uploading multiple batches */} + {isUploading && ( + + + Uploading products: {uploadProgress}% + + + + )} {uploadError && ( {uploadError}