[Express][TypeScript] Uploading file 1

Intro

When I upload large files in ASP.NET Core applications, I slice the data and send them, after finishing uploading all sliced data, I merge them.

[Angular][ASP.NET Core] Upload chunked files

How about in Node.js applicaitons?

I will try it.

Environments

Node.js ver.16.3.0

TypeScript ver.4.3.2

Server-side

ts-node ver.10.0.0

Express ver.4.17.1

moment ver.2.29.1

Client-side

Webpack ver.5.39.1

ts-loader ver.9.2.3

webpack-cli ver.4.7.2

Upload ReadableStream (failed)

Because both client-side and server-side JavaScript had had "Stream", so I thought I could send file data by them.

[Client] index.html

<!DOCTYPE html>
<html lang='en'>
    <head>
        <title>Hello</title>
        <meta charset="utf8">
    </head>
    <body>
        <input type="file" id="upload_file_input">
        <button onclick="Page.upload()">Upload</button>
        <script src="./js/main.page.js"></script>
    </body>
</html>

[Client] main.page.ts

export async function upload() {
    const fileElement = document.getElementById('upload_file_input') as HTMLInputElement;
    if(fileElement?.files == null ||
        fileElement?.files.length <= 0) {
        alert('No any files');
        return;
    }
    const readData = await fileElement.files[0].arrayBuffer();
    if(readData == null || readData.byteLength <= 0) {
        return 'Failed reading';
    }
    const buffer = new Uint8Array(readData);
    const stream = new ReadableStream({
        pull: (controller) => {            
            let index = 0;
            let chunkSize = 100;
            function push() {         
                if(buffer.byteLength <= index + 1) {
                    controller.close();
                    return;
                }
                if(buffer.byteLength <= index + chunkSize) {
                    chunkSize = buffer.byteLength - index - 1;
                }
                controller.enqueue(new Blob([buffer.subarray(index, index + chunkSize)]));
                index += chunkSize;
                push();
            }
            return push();
        },
    });

    const response = await fetch('files', {
        method: 'POST',
        mode: 'cors',
        headers: {
            'Content-Type': 'application/octet-stream'
        },
        body: stream
    });
    if(response.ok) {
        console.log('OK');
        console.log(await response.text());
    } else {
        console.error('Failed');
    }
}

[Server] index.ts

import express from 'express';
import fs from 'fs';

const port = 3000;
const app = express();
app.use(express.static('clients/public'));
app.post('/files', (req, res, next) => {
    let buffer: Buffer|null = null;
    req.on('data', (chunk) => {
        if(buffer == null) {
            buffer = Buffer.from(chunk);
        } else {
            buffer = Buffer.concat([buffer, chunk]);
        }
    });

    req.on('end', () => {          
        fs.writeFile('sample.png', buffer as Buffer, err => {
            console.log(err);
        });
        next();
    });
    res.send('OK');
});

app.listen(port, () => {
    console.log(`Example app listening at http://localhost:${port}`)
});

ReadableStream - Web APIs | MDN

Using readable streams - Web APIs | MDN

Streaming requests with the fetch API - web.dev

Streams API - Web APIs | MDN

Express - Node.js web application framework

The problem was the "data" event of request of "app.post('/files')" was fired only one time.
Maybe it was because I hadn't been able to send Readable stream as request body in the Web brownsers(ex. Edge, Firefox).

w3c-test.org/fetch/api/basic/request-upload.any.html

Uploading single file

I also tried uploading the file directly.

[Client] main.page.ts

export async function upload() {
    const fileElement = document.getElementById('upload_file_input') as HTMLInputElement;
    if(fileElement?.files == null ||
        fileElement?.files.length <= 0) {
        alert('No any files');
        return;
    }    
    const response = await fetch('files', {
        method: 'POST',
        mode: 'cors',
        headers: {
            'Content-Type': 'application/octet-stream'
        },
        body: fileElement.files[0]
    });
    if(response.ok) {
        console.log('OK');
        console.log(await response.text());
    } else {
        console.error('Failed');
    }
}

The "data" event was called two or more times and I could save the uploaded file.
I also could use converted data(ex. ArrayBuffer, Uint8Array).

So when I need uploading large files, I can slice them and upload as same as in ASP.NET Core applications.

Upload sliced files

Now I try uploading sliced files in the Express application.

Specs

[Client] Select a file.

[Client] Slice 1. to small(1KB) blobs.

[Client] Send the file name to the Server-side on starting uploading.

[Server] Create a directory for saving sliced blobs.

[Client] Send sliced blobs.

[Server] Received 5. and save into 4.

[Client] Finish uploading all sliced blobs.

[Server] Merged all sliced blobs and generate a file

[Server] Remove 4.

file.types.ts

export type ActionResult = {
    succeeded: boolean,
    errorMessage: string,
}

[Client] main.page.ts

import { ActionResult } from "./file.types";

export async function upload() {
    const fileElement = document.getElementById('upload_file_input') as HTMLInputElement;
    if(fileElement?.files == null ||
        fileElement?.files.length <= 0) {
        alert('No any files');
        return;
    }
    const readData = await fileElement.files[0].arrayBuffer();
    if(readData == null || readData.byteLength <= 0) {
        return 'Failed reading';
    }
    const startResult = await startUploading(fileElement.files[0].name);
    if(startResult.result.succeeded === false) {
        alert(startResult.result.errorMessage);
        return;
    }
    const uploadResult = await uploadChunks(readData, startResult.folderName);
    if(uploadResult.succeeded === false) {
        alert(uploadResult.errorMessage);
        return;
    }
    const postResult = await postUploading(startResult.folderName, fileElement.files[0].name);
    if(postResult.succeeded === false) {
        alert(postResult.errorMessage);
        return;
    }
    alert('OK');
}
async function startUploading(fileName: string): Promise<{ result: ActionResult, folderName: string }> {
    const startResponse = await fetch('files/start', {
        method: 'POST',
        mode: 'cors',
        headers: {
            'Content-Type': 'application/json'
        },
        body: JSON.stringify({ fileName }),
    });
    const responseJson = await startResponse.json();
    return JSON.parse(JSON.stringify(responseJson))
}
async function uploadChunks(fileData: ArrayBuffer, folderName: string): Promise<ActionResult> {
    let index = 0;
    let chunkSize = 1024;
    const buffer = new Uint8Array(fileData);
    while(true) {
        if(buffer.byteLength <= index + 1) {
            console.log('end');

            return { succeeded: true, errorMessage: '' };
        }
        if(buffer.byteLength <= index + chunkSize) {
            chunkSize = fileData.byteLength - index - 1;
        }
        const response = await fetch('files/chunk', {
            method: 'POST',
            mode: 'cors',
            headers: {
                'folderName': folderName,
                'index': index.toString(),
                'Content-Type': 'application/octet-stream'
            },
            body: new Blob([buffer.subarray(index, index + chunkSize)])
        });
        const responseJson = await response.json();
        const result = JSON.parse(JSON.stringify(responseJson)) as ActionResult;
        if(result.succeeded === false) {
            return result;
        }
        index += chunkSize;
    }
}
async function postUploading(folderName: string, fileName: string): Promise<ActionResult> {
    const response = await fetch('files/end', {
        method: 'POST',
        mode: 'cors',
        headers: {
            'Content-Type': 'application/json'
        },
        body: JSON.stringify({ folderName, fileName }),
    });
    const responseJson = await response.json();
    return JSON.parse(JSON.stringify(responseJson)); 
}

[Server] index.ts

import express from 'express';
import fs from 'fs';
import moment from 'moment';
import * as actionResults from './actionResultFactory';

const port = 3000;
const app = express();
// To receive JSON value from client-side
app.use(express.json());
// To receive Blob value from client-side
app.use(express.raw());
app.use(express.static('clients/public'));
app.post('/files/start', async (req, res) => {
    const startUploading = JSON.parse(JSON.stringify(req.body));
    const folderName = await createDirectory(startUploading.fileName);
    res.json({
        result: actionResults.getSucceeded(),
        folderName,
    });
});
app.post('/files/chunk', (req, res) => {
    const itemIndex = req.headers['index'];
    const saveDirectory = req.headers['foldername'];
    if(itemIndex == null ||
        saveDirectory == null) {
        res.json(actionResults.getFailed('No data'));
        return;
    }
    fs.promises.writeFile(`tmp/${saveDirectory}/${itemIndex}_value`, Buffer.from(req.body))
        .then(_ => res.json(actionResults.getSucceeded()))
        .catch(err => res.json(actionResults.getFailed(err)));
});
app.post('/files/end', async (req, res) => {
    const savedTmpFiles = JSON.parse(JSON.stringify(req.body));
    const savedDirectory = `tmp/${savedTmpFiles.folderName}`;
    const dirs = await fs.promises.readdir(savedDirectory, { withFileTypes: true });
    const files = dirs.filter(d => /^[0-9]+_value$/).map(d => {
        return { index: parseInt((d.name.split('_')[0])), name: d.name}
    });
    let buffer: Buffer|null = null;
    for(const d of files.sort((a, b) => a.index - b.index)) {
        var newBuffer = Buffer.from(await fs.promises.readFile(`${savedDirectory}/${d.name}`));
        if(buffer == null) {
            buffer = newBuffer;
        } else {
            buffer = Buffer.concat([buffer, newBuffer]);
        }
    }
    fs.promises.writeFile(`tmp/${savedTmpFiles.fileName}`, buffer as Buffer)
        .then(_ => fs.promises.rm(savedDirectory, { force: true, recursive: true }))
        .then(_ => res.json(actionResults.getSucceeded()))
        .catch(err => res.json(actionResults.getFailed(err)));
});

app.listen(port, () => {
    console.log(`Example app listening at http://localhost:${port}`)
});
async function createDirectory(fileName: string): Promise<string> {
    if((await exists('tmp')) === false) {
        await fs.promises.mkdir('tmp');
    }
    const folderName = `${moment(Date.now()).format('YYYYMMDDHHmmssfff')}_${fileName}`;
    await fs.promises.mkdir(`tmp/${folderName}`);
    return folderName;
}
async function exists(path: string): Promise<boolean> {
    return new Promise(async (resolve) => {
        fs.promises.stat(path)
            .then(s => resolve(true))
            .catch(err => resolve(false));
    });
}