Batched bulk uploads
This commit is contained in:
parent
a06863562c
commit
7bd4f3de0e
3 changed files with 233 additions and 174 deletions
|
|
@ -157,7 +157,8 @@ app.use(cors({
|
|||
}));
|
||||
|
||||
app.use('/api/payment', stripePaymentRoutes(pool, query, authMiddleware(pool, query)));
|
||||
app.use(express.json());
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
app.use(express.urlencoded({ extended: true, limit: '10mb' }));
|
||||
app.use(morgan('dev'));
|
||||
|
||||
// Serve static files - serve the entire public directory
|
||||
|
|
|
|||
|
|
@ -4,183 +4,182 @@
|
|||
-- Note: Product categories are already inserted in the schema file, so we're skipping that step
|
||||
|
||||
-- Insert rock products
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
-- Insert rock products
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
|
||||
SELECT
|
||||
'Amethyst Geode',
|
||||
'Beautiful purple amethyst geode with crystal formation',
|
||||
id,
|
||||
49.99,
|
||||
15,
|
||||
450.0,
|
||||
'Purple',
|
||||
'Quartz',
|
||||
'Brazil',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Rock';
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
|
||||
-- SELECT
|
||||
-- 'Amethyst Geode',
|
||||
-- 'Beautiful purple amethyst geode with crystal formation',
|
||||
-- id,
|
||||
-- 49.99,
|
||||
-- 15,
|
||||
-- 450.0,
|
||||
-- 'Purple',
|
||||
-- 'Quartz',
|
||||
-- 'Brazil',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Rock';
|
||||
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
|
||||
SELECT
|
||||
'Polished Labradorite',
|
||||
'Stunning polished labradorite with iridescent blue flash',
|
||||
id,
|
||||
29.99,
|
||||
25,
|
||||
120.5,
|
||||
'Gray/Blue',
|
||||
'Feldspar',
|
||||
'Madagascar',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Rock';
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
|
||||
-- SELECT
|
||||
-- 'Polished Labradorite',
|
||||
-- 'Stunning polished labradorite with iridescent blue flash',
|
||||
-- id,
|
||||
-- 29.99,
|
||||
-- 25,
|
||||
-- 120.5,
|
||||
-- 'Gray/Blue',
|
||||
-- 'Feldspar',
|
||||
-- 'Madagascar',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Rock';
|
||||
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
|
||||
SELECT
|
||||
'Raw Turquoise',
|
||||
'Natural turquoise specimen, unpolished',
|
||||
id,
|
||||
19.99,
|
||||
18,
|
||||
85.2,
|
||||
'Turquoise',
|
||||
'Turquoise',
|
||||
'Arizona',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Rock';
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, weight_grams, color, material_type, origin, image_url)
|
||||
-- SELECT
|
||||
-- 'Raw Turquoise',
|
||||
-- 'Natural turquoise specimen, unpolished',
|
||||
-- id,
|
||||
-- 19.99,
|
||||
-- 18,
|
||||
-- 85.2,
|
||||
-- 'Turquoise',
|
||||
-- 'Turquoise',
|
||||
-- 'Arizona',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Rock';
|
||||
|
||||
-- Insert bone products
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
|
||||
SELECT
|
||||
'Deer Antler',
|
||||
'Naturally shed deer antler, perfect for display or crafts',
|
||||
id,
|
||||
24.99,
|
||||
8,
|
||||
38.5,
|
||||
'Antler',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Bone';
|
||||
-- -- Insert bone products
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
|
||||
-- SELECT
|
||||
-- 'Deer Antler',
|
||||
-- 'Naturally shed deer antler, perfect for display or crafts',
|
||||
-- id,
|
||||
-- 24.99,
|
||||
-- 8,
|
||||
-- 38.5,
|
||||
-- 'Antler',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Bone';
|
||||
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
|
||||
SELECT
|
||||
'Fossil Fish',
|
||||
'Well-preserved fossil fish from the Green River Formation',
|
||||
id,
|
||||
89.99,
|
||||
5,
|
||||
22.8,
|
||||
'Fossilized Bone',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Bone';
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, material_type, image_url)
|
||||
-- SELECT
|
||||
-- 'Fossil Fish',
|
||||
-- 'Well-preserved fossil fish from the Green River Formation',
|
||||
-- id,
|
||||
-- 89.99,
|
||||
-- 5,
|
||||
-- 22.8,
|
||||
-- 'Fossilized Bone',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Bone';
|
||||
|
||||
-- Insert stick products
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
|
||||
SELECT
|
||||
'Driftwood Piece',
|
||||
'Unique driftwood piece, weathered by the ocean',
|
||||
id,
|
||||
14.99,
|
||||
12,
|
||||
45.6,
|
||||
8.3,
|
||||
'Driftwood',
|
||||
'Tan/Gray',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Stick';
|
||||
-- -- Insert stick products
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
|
||||
-- SELECT
|
||||
-- 'Driftwood Piece',
|
||||
-- 'Unique driftwood piece, weathered by the ocean',
|
||||
-- id,
|
||||
-- 14.99,
|
||||
-- 12,
|
||||
-- 45.6,
|
||||
-- 8.3,
|
||||
-- 'Driftwood',
|
||||
-- 'Tan/Gray',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Stick';
|
||||
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
|
||||
SELECT
|
||||
'Walking Stick',
|
||||
'Hand-selected natural maple walking stick',
|
||||
id,
|
||||
34.99,
|
||||
10,
|
||||
152.4,
|
||||
3.8,
|
||||
'Maple',
|
||||
'Brown',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Stick';
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
|
||||
-- SELECT
|
||||
-- 'Walking Stick',
|
||||
-- 'Hand-selected natural maple walking stick',
|
||||
-- id,
|
||||
-- 34.99,
|
||||
-- 10,
|
||||
-- 152.4,
|
||||
-- 3.8,
|
||||
-- 'Maple',
|
||||
-- 'Brown',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Stick';
|
||||
|
||||
WITH categories AS (
|
||||
SELECT id, name FROM product_categories
|
||||
)
|
||||
INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
|
||||
SELECT
|
||||
'Decorative Branch Set',
|
||||
'Set of 3 decorative birch branches for home decoration',
|
||||
id,
|
||||
19.99,
|
||||
20,
|
||||
76.2,
|
||||
1.5,
|
||||
'Birch',
|
||||
'White',
|
||||
NULL
|
||||
FROM categories WHERE name = 'Stick';
|
||||
-- WITH categories AS (
|
||||
-- SELECT id, name FROM product_categories
|
||||
-- )
|
||||
-- INSERT INTO products (name, description, category_id, price, stock_quantity, length_cm, width_cm, material_type, color, image_url)
|
||||
-- SELECT
|
||||
-- 'Decorative Branch Set',
|
||||
-- 'Set of 3 decorative birch branches for home decoration',
|
||||
-- id,
|
||||
-- 19.99,
|
||||
-- 20,
|
||||
-- 76.2,
|
||||
-- 1.5,
|
||||
-- 'Birch',
|
||||
-- 'White',
|
||||
-- NULL
|
||||
-- FROM categories WHERE name = 'Stick';
|
||||
|
||||
-- Create a cart for testing
|
||||
INSERT INTO carts (user_id)
|
||||
SELECT id FROM users WHERE email = 'jane@example.com';
|
||||
-- -- Create a cart for testing
|
||||
-- INSERT INTO carts (user_id)
|
||||
-- SELECT id FROM users WHERE email = 'jane@example.com';
|
||||
|
||||
-- Add product tags - using a different approach
|
||||
-- Tag: Decorative for Amethyst Geode
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative';
|
||||
-- -- Add product tags - using a different approach
|
||||
-- -- Tag: Decorative for Amethyst Geode
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Amethyst Geode' AND t.name = 'Decorative';
|
||||
|
||||
-- Tag: Polished for Polished Labradorite
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished';
|
||||
-- -- Tag: Polished for Polished Labradorite
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Polished Labradorite' AND t.name = 'Polished';
|
||||
|
||||
-- Tag: Raw for Raw Turquoise
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw';
|
||||
-- -- Tag: Raw for Raw Turquoise
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Raw Turquoise' AND t.name = 'Raw';
|
||||
|
||||
-- Tags: Fossil and Educational for Fossil Fish
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil';
|
||||
-- -- Tags: Fossil and Educational for Fossil Fish
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Fossil Fish' AND t.name = 'Fossil';
|
||||
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Fossil Fish' AND t.name = 'Educational';
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Fossil Fish' AND t.name = 'Educational';
|
||||
|
||||
-- Tag: Decorative for Driftwood Piece
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative';
|
||||
-- -- Tag: Decorative for Driftwood Piece
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Driftwood Piece' AND t.name = 'Decorative';
|
||||
|
||||
-- Tag: Collectible for Walking Stick
|
||||
INSERT INTO product_tags (product_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM products p, tags t
|
||||
WHERE p.name = 'Walking Stick' AND t.name = 'Collectible';
|
||||
-- -- Tag: Collectible for Walking Stick
|
||||
-- INSERT INTO product_tags (product_id, tag_id)
|
||||
-- SELECT p.id, t.id
|
||||
-- FROM products p, tags t
|
||||
-- WHERE p.name = 'Walking Stick' AND t.name = 'Collectible';
|
||||
|
|
@ -23,7 +23,8 @@ import {
|
|||
DialogContentText,
|
||||
DialogTitle,
|
||||
Stack,
|
||||
Tooltip
|
||||
Tooltip,
|
||||
LinearProgress
|
||||
} from '@mui/material';
|
||||
import {
|
||||
Edit as EditIcon,
|
||||
|
|
@ -468,7 +469,7 @@ const AdminProductsPage = () => {
|
|||
};
|
||||
|
||||
// Submit bulk upload
|
||||
const handleSubmitUpload = () => {
|
||||
const handleSubmitUpload = async () => {
|
||||
if (parsedProducts.length === 0) {
|
||||
setUploadError('No valid products found in the file. Please check your data.');
|
||||
return;
|
||||
|
|
@ -477,19 +478,64 @@ const AdminProductsPage = () => {
|
|||
setIsUploading(true);
|
||||
setUploadError(null);
|
||||
|
||||
// Submit products in batches of 20 to respect connection limits
|
||||
const batchSize = 20;
|
||||
// Process products in batches to avoid payload size limits
|
||||
const batchSize = 50; // Smaller batch size to avoid payload limits
|
||||
const batches = [];
|
||||
|
||||
for (let i = 0; i < parsedProducts.length; i += batchSize) {
|
||||
const batch = parsedProducts.slice(i, i + batchSize);
|
||||
batches.push(batch);
|
||||
batches.push(parsedProducts.slice(i, i + batchSize));
|
||||
}
|
||||
|
||||
// Process first batch
|
||||
bulkUploadProducts.mutate(parsedProducts);
|
||||
try {
|
||||
// Track successes and errors across all batches
|
||||
let allSuccesses = [];
|
||||
let allErrors = [];
|
||||
|
||||
// Process batches sequentially to avoid overwhelming the server
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
const batch = batches[i];
|
||||
setUploadProgress(Math.round((i / batches.length) * 100));
|
||||
|
||||
try {
|
||||
// Send this batch to the server
|
||||
const response = await apiClient.post('/admin/products', { products: batch });
|
||||
|
||||
// Accumulate successes and errors
|
||||
if (response.data.success) allSuccesses = [...allSuccesses, ...response.data.success];
|
||||
if (response.data.errs) allErrors = [...allErrors, ...response.data.errs];
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Error processing batch ${i+1}:`, error);
|
||||
// Add error for each product in the failed batch
|
||||
const batchErrors = batch.map(product => ({
|
||||
error: true,
|
||||
message: `Server error: ${error.message || 'Unknown error'}`,
|
||||
code: error.response?.status || 500,
|
||||
product: { name: product.name }
|
||||
}));
|
||||
allErrors = [...allErrors, ...batchErrors];
|
||||
}
|
||||
}
|
||||
|
||||
// All batches processed - invalidate product query to refresh the list
|
||||
queryClient.invalidateQueries({ queryKey: ['admin-products'] });
|
||||
|
||||
// Show results to user
|
||||
setUploadResults({
|
||||
success: allSuccesses,
|
||||
errs: allErrors
|
||||
});
|
||||
|
||||
setUploadSuccess(true);
|
||||
} catch (error) {
|
||||
setUploadError(`Failed to upload products: ${error.message}`);
|
||||
} finally {
|
||||
setIsUploading(false);
|
||||
setUploadProgress(0);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Filter and paginate products
|
||||
const filteredProducts = products || [];
|
||||
const paginatedProducts = filteredProducts.slice(
|
||||
|
|
@ -819,6 +865,19 @@ const AdminProductsPage = () => {
|
|||
</Alert>
|
||||
)}
|
||||
|
||||
{/* Show progress bar when uploading multiple batches */}
|
||||
{isUploading && (
|
||||
<Box sx={{ width: '100%', mt: 2 }}>
|
||||
<Typography variant="body2" color="text.secondary" sx={{ mb: 1 }}>
|
||||
Uploading products: {uploadProgress}%
|
||||
</Typography>
|
||||
<LinearProgress
|
||||
variant="determinate"
|
||||
value={uploadProgress}
|
||||
sx={{ height: 10, borderRadius: 1 }}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
{uploadError && (
|
||||
<Alert severity="error" sx={{ mt: 2, width: '100%' }}>
|
||||
{uploadError}
|
||||
|
|
|
|||
Loading…
Reference in a new issue