Batched bulk uploads

backend/src/index.js

@@ -157,7 +157,8 @@ app.use(cors({
 }));
 
 app.use('/api/payment', stripePaymentRoutes(pool, query, authMiddleware(pool, query)));
-app.use(express.json());
+app.use(express.json({ limit: '10mb' }));
+app.use(express.urlencoded({ extended: true, limit: '10mb' }));
 app.use(morgan('dev'));
 
 // Serve static files - serve the entire public directory

db/init/02-seed.sql

@@ -4,183 +4,182 @@
 -- Note: Product categories are already inserted in the schema file, so we're skipping that step
 
 -- Insert rock products
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
--- Insert rock products
-INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
-SELECT 
-  'Amethyst Geode', 
-  'Beautiful purple amethyst geode with crystal formation', 
-  id, 
-  49.99, 
-  15, 
-  450.0, 
-  'Purple', 
-  'Quartz', 
-  'Brazil', 
-  NULL
-FROM categories WHERE name = 'Rock';
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
+-- SELECT 
+--   'Amethyst Geode', 
+--   'Beautiful purple amethyst geode with crystal formation', 
+--   id, 
+--   49.99, 
+--   15, 
+--   450.0, 
+--   'Purple', 
+--   'Quartz', 
+--   'Brazil', 
+--   NULL
+-- FROM categories WHERE name = 'Rock';
 
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
-SELECT 
-  'Polished Labradorite', 
-  'Stunning polished labradorite with iridescent blue flash', 
-  id, 
-  29.99, 
-  25, 
-  120.5, 
-  'Gray/Blue', 
-  'Feldspar', 
-  'Madagascar',  
-  NULL
-FROM categories WHERE name = 'Rock';
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
+-- SELECT 
+--   'Polished Labradorite', 
+--   'Stunning polished labradorite with iridescent blue flash', 
+--   id, 
+--   29.99, 
+--   25, 
+--   120.5, 
+--   'Gray/Blue', 
+--   'Feldspar', 
+--   'Madagascar',  
+--   NULL
+-- FROM categories WHERE name = 'Rock';
 
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
-SELECT 
-  'Raw Turquoise', 
-  'Natural turquoise specimen, unpolished', 
-  id, 
-  19.99, 
-  18, 
-  85.2, 
-  'Turquoise', 
-  'Turquoise', 
-  'Arizona',  
-  NULL
-FROM categories WHERE name = 'Rock';
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
+-- SELECT 
+--   'Raw Turquoise', 
+--   'Natural turquoise specimen, unpolished', 
+--   id, 
+--   19.99, 
+--   18, 
+--   85.2, 
+--   'Turquoise', 
+--   'Turquoise', 
+--   'Arizona',  
+--   NULL
+-- FROM categories WHERE name = 'Rock';
 
--- Insert bone products
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
-SELECT 
-  'Deer Antler', 
-  'Naturally shed deer antler, perfect for display or crafts', 
-  id, 
-  24.99, 
-  8, 
-  38.5, 
-  'Antler',  
-  NULL
-FROM categories WHERE name = 'Bone';
+-- -- Insert bone products
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
+-- SELECT 
+--   'Deer Antler', 
+--   'Naturally shed deer antler, perfect for display or crafts', 
+--   id, 
+--   24.99, 
+--   8, 
+--   38.5, 
+--   'Antler',  
+--   NULL
+-- FROM categories WHERE name = 'Bone';
 
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
-SELECT 
-  'Fossil Fish', 
-  'Well-preserved fossil fish from the Green River Formation', 
-  id, 
-  89.99, 
-  5, 
-  22.8, 
-  'Fossilized Bone',  
-  NULL
-FROM categories WHERE name = 'Bone';
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
+-- SELECT 
+--   'Fossil Fish', 
+--   'Well-preserved fossil fish from the Green River Formation', 
+--   id, 
+--   89.99, 
+--   5, 
+--   22.8, 
+--   'Fossilized Bone',  
+--   NULL
+-- FROM categories WHERE name = 'Bone';
 
--- Insert stick products
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
-SELECT 
-  'Driftwood Piece', 
-  'Unique driftwood piece, weathered by the ocean', 
-  id, 
-  14.99, 
-  12, 
-  45.6, 
-  8.3, 
-  'Driftwood', 
-  'Tan/Gray',  
-  NULL
-FROM categories WHERE name = 'Stick';
+-- -- Insert stick products
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
+-- SELECT 
+--   'Driftwood Piece', 
+--   'Unique driftwood piece, weathered by the ocean', 
+--   id, 
+--   14.99, 
+--   12, 
+--   45.6, 
+--   8.3, 
+--   'Driftwood', 
+--   'Tan/Gray',  
+--   NULL
+-- FROM categories WHERE name = 'Stick';
 
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
-SELECT 
-  'Walking Stick', 
-  'Hand-selected natural maple walking stick', 
-  id, 
-  34.99, 
-  10, 
-  152.4, 
-  3.8, 
-  'Maple', 
-  'Brown',  
-  NULL
-FROM categories WHERE name = 'Stick';
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
+-- SELECT 
+--   'Walking Stick', 
+--   'Hand-selected natural maple walking stick', 
+--   id, 
+--   34.99, 
+--   10, 
+--   152.4, 
+--   3.8, 
+--   'Maple', 
+--   'Brown',  
+--   NULL
+-- FROM categories WHERE name = 'Stick';
 
-WITH categories AS (
-  SELECT id, name FROM product_categories
-)
-INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
-SELECT 
-  'Decorative Branch Set', 
-  'Set of 3 decorative birch branches for home decoration', 
-  id, 
-  19.99, 
-  20, 
-  76.2, 
-  1.5, 
-  'Birch', 
-  'White',  
-  NULL
-FROM categories WHERE name = 'Stick';
+-- WITH categories AS (
+--   SELECT id, name FROM product_categories
+-- )
+-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
+-- SELECT 
+--   'Decorative Branch Set', 
+--   'Set of 3 decorative birch branches for home decoration', 
+--   id, 
+--   19.99, 
+--   20, 
+--   76.2, 
+--   1.5, 
+--   'Birch', 
+--   'White',  
+--   NULL
+-- FROM categories WHERE name = 'Stick';
 
--- Create a cart for testing
-INSERT INTO carts (user_id)
-SELECT id FROM users WHERE email = 'jane@example.com';
+-- -- Create a cart for testing
+-- INSERT INTO carts (user_id)
+-- SELECT id FROM users WHERE email = 'jane@example.com';
 
--- Add product tags - using a different approach
--- Tag: Decorative for Amethyst Geode
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative';
+-- -- Add product tags - using a different approach
+-- -- Tag: Decorative for Amethyst Geode
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative';
 
--- Tag: Polished for Polished Labradorite
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished';
+-- -- Tag: Polished for Polished Labradorite
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished';
 
--- Tag: Raw for Raw Turquoise
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw';
+-- -- Tag: Raw for Raw Turquoise
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw';
 
--- Tags: Fossil and Educational for Fossil Fish
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil';
+-- -- Tags: Fossil and Educational for Fossil Fish
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil';
 
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Fossil Fish' AND t.name = 'Educational';
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Fossil Fish' AND t.name = 'Educational';
 
--- Tag: Decorative for Driftwood Piece
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative';
+-- -- Tag: Decorative for Driftwood Piece
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative';
 
--- Tag: Collectible for Walking Stick
-INSERT INTO product_tags (product_id, tag_id)
-SELECT p.id, t.id 
-FROM products p, tags t
-WHERE p.name = 'Walking Stick' AND t.name = 'Collectible';
+-- -- Tag: Collectible for Walking Stick
+-- INSERT INTO product_tags (product_id, tag_id)
+-- SELECT p.id, t.id 
+-- FROM products p, tags t
+-- WHERE p.name = 'Walking Stick' AND t.name = 'Collectible';

frontend/src/pages/Admin/ProductsPage.jsx

@@ -23,7 +23,8 @@ import {
   DialogContentText,
   DialogTitle,
   Stack,
-  Tooltip
+  Tooltip,
+  LinearProgress
 } from '@mui/material';
 import {
   Edit as EditIcon,
@@ -468,7 +469,7 @@ const AdminProductsPage = () => {
   };
   
   // Submit bulk upload
-  const handleSubmitUpload = () => {
+  const handleSubmitUpload = async () => {
     if (parsedProducts.length === 0) {
       setUploadError('No valid products found in the file. Please check your data.');
       return;
@@ -477,19 +478,64 @@ const AdminProductsPage = () => {
     setIsUploading(true);
     setUploadError(null);
     
-    // Submit products in batches of 20 to respect connection limits
-    const batchSize = 20;
+    // Process products in batches to avoid payload size limits
+    const batchSize = 50; // Smaller batch size to avoid payload limits
     const batches = [];
     
     for (let i = 0; i < parsedProducts.length; i += batchSize) {
-      const batch = parsedProducts.slice(i, i + batchSize);
-      batches.push(batch);
+      batches.push(parsedProducts.slice(i, i + batchSize));
     }
     
-    // Process first batch
-    bulkUploadProducts.mutate(parsedProducts);
+    try {
+      // Track successes and errors across all batches
+      let allSuccesses = [];
+      let allErrors = [];
+      
+      // Process batches sequentially to avoid overwhelming the server
+      for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i];
+        setUploadProgress(Math.round((i / batches.length) * 100));
+        
+        try {
+          // Send this batch to the server
+          const response = await apiClient.post('/admin/products', { products: batch });
+          
+          // Accumulate successes and errors
+          if (response.data.success) allSuccesses = [...allSuccesses, ...response.data.success];
+          if (response.data.errs) allErrors = [...allErrors, ...response.data.errs];
+          
+        } catch (error) {
+          console.error(`Error processing batch ${i+1}:`, error);
+          // Add error for each product in the failed batch
+          const batchErrors = batch.map(product => ({
+            error: true,
+            message: `Server error: ${error.message || 'Unknown error'}`,
+            code: error.response?.status || 500,
+            product: { name: product.name }
+          }));
+          allErrors = [...allErrors, ...batchErrors];
+        }
+      }
+      
+      // All batches processed - invalidate product query to refresh the list
+      queryClient.invalidateQueries({ queryKey: ['admin-products'] });
+      
+      // Show results to user
+      setUploadResults({
+        success: allSuccesses,
+        errs: allErrors
+      });
+      
+      setUploadSuccess(true);
+    } catch (error) {
+      setUploadError(`Failed to upload products: ${error.message}`);
+    } finally {
+      setIsUploading(false);
+      setUploadProgress(0);
+    }
   };
 
+
   // Filter and paginate products
   const filteredProducts = products || [];
   const paginatedProducts = filteredProducts.slice(
@@ -819,6 +865,19 @@ const AdminProductsPage = () => {
                   </Alert>
                 )}
                 
+                {/* Show progress bar when uploading multiple batches */}
+                {isUploading && (
+                  <Box sx={{ width: '100%', mt: 2 }}>
+                    <Typography variant="body2" color="text.secondary" sx={{ mb: 1 }}>
+                      Uploading products: {uploadProgress}%
+                    </Typography>
+                    <LinearProgress 
+                      variant="determinate" 
+                      value={uploadProgress} 
+                      sx={{ height: 10, borderRadius: 1 }}
+                    />
+                  </Box>
+                )}
                 {uploadError && (
                   <Alert severity="error" sx={{ mt: 2, width: '100%' }}>
                     {uploadError}