root/E-Commerce-Module
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Batched bulk uploads

Browse source
This commit is contained in:
2ManyProjects 2025-05-05 17:48:46 -05:00
parent a06863562c
commit 7bd4f3de0e
3 changed files with 233 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/src/index.js
View file

@ -157,7 +157,8 @@ app.use(cors({
})); }));
app.use('/api/payment', stripePaymentRoutes(pool, query, authMiddleware(pool, query))); app.use('/api/payment', stripePaymentRoutes(pool, query, authMiddleware(pool, query)));
app.use(express.json()); app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true, limit: '10mb' }));
app.use(morgan('dev')); app.use(morgan('dev'));
// Serve static files - serve the entire public directory // Serve static files - serve the entire public directory

329
db/init/02-seed.sql
View file

@ -4,183 +4,182 @@
-- Note: Product categories are already inserted in the schema file, so we're skipping that step -- Note: Product categories are already inserted in the schema file, so we're skipping that step
-- Insert rock products -- Insert rock products
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
-- Insert rock products -- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) -- SELECT
SELECT -- 'Amethyst Geode',
'Amethyst Geode', -- 'Beautiful purple amethyst geode with crystal formation',
'Beautiful purple amethyst geode with crystal formation', -- id,
id, -- 49.99,
49.99, -- 15,
15, -- 450.0,
450.0, -- 'Purple',
'Purple', -- 'Quartz',
'Quartz', -- 'Brazil',
'Brazil', -- NULL
NULL -- FROM categories WHERE name = 'Rock';
FROM categories WHERE name = 'Rock';
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
SELECT -- SELECT
'Polished Labradorite', -- 'Polished Labradorite',
'Stunning polished labradorite with iridescent blue flash', -- 'Stunning polished labradorite with iridescent blue flash',
id, -- id,
29.99, -- 29.99,
25, -- 25,
120.5, -- 120.5,
'Gray/Blue', -- 'Gray/Blue',
'Feldspar', -- 'Feldspar',
'Madagascar', -- 'Madagascar',
NULL -- NULL
FROM categories WHERE name = 'Rock'; -- FROM categories WHERE name = 'Rock';
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
SELECT -- SELECT
'Raw Turquoise', -- 'Raw Turquoise',
'Natural turquoise specimen, unpolished', -- 'Natural turquoise specimen, unpolished',
id, -- id,
19.99, -- 19.99,
18, -- 18,
85.2, -- 85.2,
'Turquoise', -- 'Turquoise',
'Turquoise', -- 'Turquoise',
'Arizona', -- 'Arizona',
NULL -- NULL
FROM categories WHERE name = 'Rock'; -- FROM categories WHERE name = 'Rock';
-- Insert bone products -- -- Insert bone products
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
SELECT -- SELECT
'Deer Antler', -- 'Deer Antler',
'Naturally shed deer antler, perfect for display or crafts', -- 'Naturally shed deer antler, perfect for display or crafts',
id, -- id,
24.99, -- 24.99,
8, -- 8,
38.5, -- 38.5,
'Antler', -- 'Antler',
NULL -- NULL
FROM categories WHERE name = 'Bone'; -- FROM categories WHERE name = 'Bone';
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
SELECT -- SELECT
'Fossil Fish', -- 'Fossil Fish',
'Well-preserved fossil fish from the Green River Formation', -- 'Well-preserved fossil fish from the Green River Formation',
id, -- id,
89.99, -- 89.99,
5, -- 5,
22.8, -- 22.8,
'Fossilized Bone', -- 'Fossilized Bone',
NULL -- NULL
FROM categories WHERE name = 'Bone'; -- FROM categories WHERE name = 'Bone';
-- Insert stick products -- -- Insert stick products
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
SELECT -- SELECT
'Driftwood Piece', -- 'Driftwood Piece',
'Unique driftwood piece, weathered by the ocean', -- 'Unique driftwood piece, weathered by the ocean',
id, -- id,
14.99, -- 14.99,
12, -- 12,
45.6, -- 45.6,
8.3, -- 8.3,
'Driftwood', -- 'Driftwood',
'Tan/Gray', -- 'Tan/Gray',
NULL -- NULL
FROM categories WHERE name = 'Stick'; -- FROM categories WHERE name = 'Stick';
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
SELECT -- SELECT
'Walking Stick', -- 'Walking Stick',
'Hand-selected natural maple walking stick', -- 'Hand-selected natural maple walking stick',
id, -- id,
34.99, -- 34.99,
10, -- 10,
152.4, -- 152.4,
3.8, -- 3.8,
'Maple', -- 'Maple',
'Brown', -- 'Brown',
NULL -- NULL
FROM categories WHERE name = 'Stick'; -- FROM categories WHERE name = 'Stick';
WITH categories AS ( -- WITH categories AS (
SELECT id, name FROM product_categories -- SELECT id, name FROM product_categories
) -- )
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url) -- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
SELECT -- SELECT
'Decorative Branch Set', -- 'Decorative Branch Set',
'Set of 3 decorative birch branches for home decoration', -- 'Set of 3 decorative birch branches for home decoration',
id, -- id,
19.99, -- 19.99,
20, -- 20,
76.2, -- 76.2,
1.5, -- 1.5,
'Birch', -- 'Birch',
'White', -- 'White',
NULL -- NULL
FROM categories WHERE name = 'Stick'; -- FROM categories WHERE name = 'Stick';
-- Create a cart for testing -- -- Create a cart for testing
INSERT INTO carts (user_id) -- INSERT INTO carts (user_id)
SELECT id FROM users WHERE email = 'jane@example.com'; -- SELECT id FROM users WHERE email = 'jane@example.com';
-- Add product tags - using a different approach -- -- Add product tags - using a different approach
-- Tag: Decorative for Amethyst Geode -- -- Tag: Decorative for Amethyst Geode
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative'; -- WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative';
-- Tag: Polished for Polished Labradorite -- -- Tag: Polished for Polished Labradorite
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished'; -- WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished';
-- Tag: Raw for Raw Turquoise -- -- Tag: Raw for Raw Turquoise
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw'; -- WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw';
-- Tags: Fossil and Educational for Fossil Fish -- -- Tags: Fossil and Educational for Fossil Fish
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil'; -- WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil';
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Fossil Fish' AND t.name = 'Educational'; -- WHERE p.name = 'Fossil Fish' AND t.name = 'Educational';
-- Tag: Decorative for Driftwood Piece -- -- Tag: Decorative for Driftwood Piece
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative'; -- WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative';
-- Tag: Collectible for Walking Stick -- -- Tag: Collectible for Walking Stick
INSERT INTO product_tags (product_id, tag_id) -- INSERT INTO product_tags (product_id, tag_id)
SELECT p.id, t.id -- SELECT p.id, t.id
FROM products p, tags t -- FROM products p, tags t
WHERE p.name = 'Walking Stick' AND t.name = 'Collectible'; -- WHERE p.name = 'Walking Stick' AND t.name = 'Collectible';

75
frontend/src/pages/Admin/ProductsPage.jsx
View file

@ -23,7 +23,8 @@ import {
DialogContentText, DialogContentText,
DialogTitle, DialogTitle,
Stack, Stack,
Tooltip Tooltip,
LinearProgress
} from '@mui/material'; } from '@mui/material';
import { import {
Edit as EditIcon, Edit as EditIcon,
@ -468,7 +469,7 @@ const AdminProductsPage = () => {
}; };
// Submit bulk upload // Submit bulk upload
const handleSubmitUpload = () => { const handleSubmitUpload = async () => {
if (parsedProducts.length === 0) { if (parsedProducts.length === 0) {
setUploadError('No valid products found in the file. Please check your data.'); setUploadError('No valid products found in the file. Please check your data.');
return; return;
@ -477,19 +478,64 @@ const AdminProductsPage = () => {
setIsUploading(true); setIsUploading(true);
setUploadError(null); setUploadError(null);
// Submit products in batches of 20 to respect connection limits // Process products in batches to avoid payload size limits
const batchSize = 20; const batchSize = 50; // Smaller batch size to avoid payload limits
const batches = []; const batches = [];
for (let i = 0; i < parsedProducts.length; i += batchSize) { for (let i = 0; i < parsedProducts.length; i += batchSize) {
const batch = parsedProducts.slice(i, i + batchSize); batches.push(parsedProducts.slice(i, i + batchSize));
batches.push(batch);
} }
// Process first batch try {
bulkUploadProducts.mutate(parsedProducts); // Track successes and errors across all batches
let allSuccesses = [];
let allErrors = [];
// Process batches sequentially to avoid overwhelming the server
for (let i = 0; i < batches.length; i++) {
const batch = batches[i];
setUploadProgress(Math.round((i / batches.length) * 100));
try {
// Send this batch to the server
const response = await apiClient.post('/admin/products', { products: batch });
// Accumulate successes and errors
if (response.data.success) allSuccesses = [...allSuccesses, ...response.data.success];
if (response.data.errs) allErrors = [...allErrors, ...response.data.errs];
} catch (error) {
console.error(`Error processing batch ${i+1}:`, error);
// Add error for each product in the failed batch
const batchErrors = batch.map(product => ({
error: true,
message: `Server error: ${error.message || 'Unknown error'}`,
code: error.response?.status || 500,
product: { name: product.name }
}));
allErrors = [...allErrors, ...batchErrors];
}
}
// All batches processed - invalidate product query to refresh the list
queryClient.invalidateQueries({ queryKey: ['admin-products'] });
// Show results to user
setUploadResults({
success: allSuccesses,
errs: allErrors
});
setUploadSuccess(true);
} catch (error) {
setUploadError(`Failed to upload products: ${error.message}`);
} finally {
setIsUploading(false);
setUploadProgress(0);
}
}; };
// Filter and paginate products // Filter and paginate products
const filteredProducts = products || []; const filteredProducts = products || [];
const paginatedProducts = filteredProducts.slice( const paginatedProducts = filteredProducts.slice(
@ -819,6 +865,19 @@ const AdminProductsPage = () => {
</Alert> </Alert>
)} )}
{/* Show progress bar when uploading multiple batches */}
{isUploading && (
<Box sx={{ width: '100%', mt: 2 }}>
<Typography variant="body2" color="text.secondary" sx={{ mb: 1 }}>
Uploading products: {uploadProgress}%
</Typography>
<LinearProgress
variant="determinate"
value={uploadProgress}
sx={{ height: 10, borderRadius: 1 }}
/>
</Box>
)}
{uploadError && ( {uploadError && (
<Alert severity="error" sx={{ mt: 2, width: '100%' }}> <Alert severity="error" sx={{ mt: 2, width: '100%' }}>
{uploadError} {uploadError}