Uploading large files to Amazon S3 can be efficiently managed using multipart uploads with presigned URLs. This approach allows you to upload file parts directly from the client-side without exposing your AWS credentials. In this article, we’ll guide you through the process of implementing S3 multipart upload using presigned URLs with Node.js on the server-side and React on the client-side.

Prerequisites

1. Node.js: Installed on your system.
2. React: A basic understanding of React and a React app set up using create-react-app.
3. AWS SDK for Node.js: Installed in your Node.js project.
4. AWS Account: Access to an AWS account with permissions to upload files to S3.

Server-Side Setup (Node.js)

First, create a new Node.js project and install the required dependencies:

mkdir s3-multipart-upload cd s3-multipart-upload npm init -y npm install @aws-sdk/client-s3 @aws-sdk/s3-request-presigner express dotenv cors body-parser

Create a .env file in the root directory to store your AWS credentials and configuration:

AWS_ACCESS_KEY_ID=your-access-key-id AWS_SECRET_ACCESS_KEY=your-secret-access-key AWS_REGION=your-region S3_BUCKET_NAME=your-bucket-name

Step 1: Initialize the Server

Create a file named server.js and initialize the server:

require('dotenv').config(); const express = require('express'); const cors = require('cors'); const bodyParser = require('body-parser'); const { S3Client, CreateMultipartUploadCommand, CompleteMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); const app = express(); app.use(cors()); app.use(bodyParser.json()); // Initialize s3Client const s3Client = new S3Client({   region: process.env.AWS_REGION,   credentials: {     accessKeyId: process.env.AWS_ACCESS_KEY_ID,     secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,   }, }); const bucketName = process.env.S3_BUCKET_NAME; app.listen(3000, () => {   console.log('Server is running on port 3000'); });

Step 2: Create Multipart Upload

Add an endpoint to create a multipart upload and get an upload ID:

const {   S3Client,   CreateMultipartUploadCommand, } = require('@aws-sdk/client-s3') app.post('/create-multipart-upload', async (req, res) => {   const { fileName } = req.body;   const params = {     Bucket: bucketName,     Key: fileName,   };   try {     const command = new CreateMultipartUploadCommand(params);     const response = await s3Client.send(command);     res.json({ uploadId: response.UploadId });   } catch (error) {     res.status(500).json({ error: error.message });   } });

Step 3: Generate Presigned URLs for Parts

Add an endpoint to generate presigned URLs for uploading parts:

const {getSignedUrl} = require('@aws-sdk/s3-request-presigner') const {   S3Client,   UploadPartCommand, } = require('@aws-sdk/client-s3') app.post('/presigned-url', async (req, res) => {   const { uploadId, parts, fileName } = req.body;   const params = {     Bucket: bucketName,     Key: fileName,     UploadId: uploadId,   }; try {   let promises = []   for (let index = 1; index <= parts; index++) {     const uploadPartCommand = new UploadPartCommand({       ...params,       PartNumber: index,     })     promises.push(getSignedUrl(s3Client, uploadPartCommand, {expiresIn: 3600}))   }   const signedUrls = await Promise.all(promises)   const partSignedUrlList = signedUrls.map((signedUrl) => {     return signedUrl   })   res.status(200).send(partSignedUrlList) } catch (error) { res.status(500).json({ error: error.message })}

Step 4: Complete Multipart Upload

Add an endpoint to complete the multipart upload:

const {   S3Client,   CompleteMultipartUploadCommand, } = require('@aws-sdk/client-s3') const _ = require('lodash') app.post('/complete-multipart-upload', async (req, res) => {   const { uploadId, fileName, parts } = req.body;   const multipartParams = {     Bucket: bucketName,     Key: fileName,     UploadId: uploadId,     MultipartUpload: {       Parts: _.orderBy(parts, ['PartNumber'], ['asc']),     },   }   let completeMultipartUploadCommand = new     CompleteMultipartUploadCommand(     multipartParams,   )   try {     await s3Client.send(completeMultipartUploadCommand)     res.status(200).json({error: error.message})   } catch (error) {     res.status(500).json({error: error.message})   } });

Client-Side Setup (React)

First, create a new React project:

npx create-react-app s3-multipart-upload-client cd s3-multipart-upload-client

Install the necessary packages:

npm install axios

Step 1: Service to Handle Multipart Upload

Create a service to handle the multipart upload logic. In src, create a file named uploadService.js:

export async function createMultipartUpload(fileName) {   const response = await axios.post(`${serverUrl}/create-multipart-upload`, { fileName });   return response.data.uploadId; }

Get Presigned urls:

export async function getPresignedUrl(uploadId, partNumber, fileName) {   const response = await axios.post(`${serverUrl}/presigned-url`, {     uploadId,     parts,     fileName,   });   return response.data.partSignedUrlList; }

Complete multipart upload:

export async function completeMultipartUpload(uploadId, fileName, parts) {   const response = await axios.post(`${serverUrl}/complete-multipart-upload`, {     uploadId,     fileName,     parts,   });   return response.data.response; }

Upload File:

export async function uploadFile(file, setProgress) {   const fileName = file.name   const uploadId = await createMultipartUpload(fileName)   const partSize = 20 * 1024 * 1024 // 20 MB   const totalParts = Math.ceil(file.size / partSize)   const presignedUrls = await getPresignedUrl(uploadId, totalParts, fileName)const parts = []   let partNumber = 1   for (let start = 0; start < file.size; start += partSize) {     const end = Math.min(start + partSize, file.size)     const blob = file.slice(start, end)     await axios.put(presignedUrls[partNumber - 1], blob, {       headers: {         'Content-Type': file.type,       },       onUploadProgress: (progressEvent) => {         const progress = Math.round(           (progressEvent.loaded * 100) / progressEvent.total,         )         setProgress((prev) => ({           ...prev,           [partNumber]: progress,         }))       },     })     parts.push({       ETag: (await axios.head(presignedUrls[index])).headers.etag,       PartNumber: partNumber,     })     partNumber++   }   return completeMultipartUpload(uploadId, fileName, parts) }

Step 2: Implement Upload Component

Create a component to handle the file input and upload logic. In src, create a new file named Upload.js:

import React, { useState } from 'react'; import { uploadFile } from './uploadService'; const Upload = () => {   const [progress, setProgress] = useState({});   const handleFileChange = async (event) => {     const file = event.target.files[0];     if (file) {       try {         const response = await uploadFile(file, setProgress);         console.log('Upload complete:', response);       } catch (error) {         console.error('Upload error:', error);       }     }   };   return (     <div>       <input type="file" onChange={handleFileChange} />       <div>         {Object.keys(progress).map((partNumber) => (           <div key={partNumber}>Part {partNumber}:   {progress[partNumber]}%</div>         ))}       </div>     </div>   ); }; export default Upload;

Step 3: Update App Component

Integrate the Upload component into your main application component. In src/App.js, update the file as follows:

import React from 'react'; import Upload from './Upload'; function App() {   return (     <div className="App">       <h1>S3 Multipart Upload</h1>       <Upload />     </div>   ); } export default App;

Step 4: Run the Application

Finally, run both the server and the React client application:

# Start the Node.js server node server.js # In a new terminal, start the React app npm start