mscdex / busboy Goto Github PK

I get following log and does nothing at var busboy = new Busboy({ headers: req.headers }); :
Regards..
Gokhan
POST /upload 200 31ms
Done parsing form!

http.js:691
throw new Error('Can't set headers after they are sent.');
^
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)

Today I switched my app from Node 0.8.25 to 0.10.26. Since then my specs tend to fail in an erratic way - for example, when I fs.stat the (piped-to) file when 'finish' fires, it is still 0 bytes in size. Any idea where to look?

It's working perfectly when i'm using normal multipart form upload. But recently I'm trying to upload file using ajax uploader. Whenever I drop my files to the inputbox i'm getting

Error: Unsupported content type: application/octet-stream
at Busboy.parseHeaders (/home/st-solutions/m3/node_modules/co-busboy/node_modules/busboy/lib/main.js:66:9)
at new Busboy (/home/st-solutions/m3/node_modules/co-busboy/node_modules/busboy/lib/main.js:21:10)
at module.exports (/home/st-solutions/m3/node_modules/co-busboy/index.js:17:16)
at Object.exports.imageSaveAction (/home/st-solutions/m3/application/controllers/productController.js:830:17)
at GeneratorFunctionPrototype.next (native)
at next (/home/st-solutions/m3/node_modules/co/index.js:74:21)
at Object. (/home/st-solutions/m3/node_modules/co/index.js:45:5)
at next (/home/st-solutions/m3/node_modules/co/index.js:90:21)
at Object. (/home/st-solutions/m3/node_modules/co/index.js:45:5)
at next (/home/st-solutions/m3/node_modules/co/index.js:90:21)

I'm trying to upload using http://filedropjs.org/demo/

Is there any solution for this?

Thanks,
Nixon

Is there a way to access the content-length of a file upload in the same way that filename and encoding can be accessed?

I'm trying to pipe the file stream from busboy to node-form-data and then back in to busboy but I keep facing "socket hung up" errors. I think it is due to me not sending a content-length to node-form-data.

Hi there,

i think we have a big backpressure leak in busboy or dicer.
If im uploading multipart-data with high-speed and have a slow filesystem (or in my case a busy GridFs) the node-memory consum raises fast! Probably we use an old v1 StreamAPI anywhere?

My testcase is simple, just upload a bigger file on localhost:

var app = require('express')();
var fs = require('fs');
var Busboy = require('busboy');

app.post('/', function(req, res) {
  console.log('GOT POST');
  var outfile = fs.createWriteStream('/dev/null');

  var busboy = new Busboy({ headers: req.headers});
  busboy.on('file', function(fieldname, stream, filename, encoding, contentType) {
    stream.pipe(outfile);
  });


  outfile.on('close', function() {
    res.send('ok');
    console.log('done');
  });
  req.pipe(busboy);
});

app.listen(3000);
console.log('listen 3000');

From the docs:

If a configured file size limit was reached, stream will both have a boolean property
truncated (best checked at the end of the stream) and emit a 'limit' event to notify you
when this happens.

I tested with a file greater than the limit and the event is not emitted.
The data is only uploaded until the given limit, but the event is not emitted.

Also I think it's strange that this event is not documented like the other ones.

Btw. the other limit events seem to work.

I'm trying to setup a Resumable.js NodeJS backend with latest Express (v4).
Whatever I upload I never get either the end or finish event.

var busboy = new Busboy({headers: req.headers});
var query = {};
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
  if(fieldname) query[fieldname] = val;
});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
  log.info('Writing chunk-%d/%d of file "%s" from GridFS', query.resumableChunkNumber, query.resumableTotalChunks, query.resumableFilename); // Called
  // var saveTo = path.join(os.tmpDir(), path.basename(fieldname));
  // file.pipe(fs.createWriteStream(saveTo));
});
busboy._parser.parser.on('finish', function() {
  d('Done parsing form!'); // Never called
});
busboy.on('finish', function() {
  d('Done parsing form!'); // Never called
});
busboy.once('end', function() {
  d('Done parsing form!'); // Never called
});
busboy.on('error', function(err) {
  d('err', err);
});
req.pipe(busboy);

Also if I try to res.send(200) on file, somehow it breaks resumable, the behavior is not the same that if I directly res.send(200), even if I have only one file.

Headers

Request URL:http://server-78b0.local:3000/upload
Request Headers
Provisional headers are shown
Accept:application/json, text/plain, */*
Authorization:Bearer eyJ0eTAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6IndhdGNoc2xhdGVAcmljaGVtb250LWNvdWNvdS5jb20iLCJmaXJzdE5hbWUiOiJXYXRjaHNsYXRlIiwibGFzdE5hbWUiOiJBcHBsaWNhdGlvbiIsInBhc3N3b3JkIjoiJDJhJDEwJHJpQXJVMHlQbDZ0MUtwUnFLZ1djbHVVVkxlODdPS2lQbjVTc2VzNzMwS21kTDZYb2JKN2pXIiwiX2lkIjoiNTNjZDFmNTlmMDAxNTIyZDA0MGI1ZWU4IiwiX192IjowfQ.0aQYQNGO2ud7DtjH5ssG_M85i53MakClhBVUQv9AEP0
Content-Type:multipart/form-data; boundary=----WebKitFormBoundaryKAbkqHZYocMBQFF4
Origin:http://localhost:9000
Referer:http://localhost:9000/
User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36
Request Payload
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableChunkNumber"

1
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableChunkSize"

1048576
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableCurrentChunkSize"

1048576
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableTotalSize"

32156861
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableType"


------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableIdentifier"

32156861-testipa
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableFilename"

test.ipa
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableRelativePath"

test.ipa
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="resumableTotalChunks"

30
------WebKitFormBoundaryKAbkqHZYocMBQFF4
Content-Disposition: form-data; name="file"; filename="blob"
Content-Type: application/octet-stream


------WebKitFormBoundaryKAbkqHZYocMBQFF4--

I cant reopen the other issue, so I opened a new one

Sadly I'm still having troubles with end not being called in some situations.
Now its happening when I don't send any file, only variables.. I really don't think I'm doing anything wrong.

Can you please test this out? Maybe I' messing with something in some other part of my code, or its express I cant know for sure.

This is not my final code, but for debuging purposes I used your example, which is having the same problem too

multipartParser.js:

var Busboy = require('busboy');

module.exports = function(req, res, next) {
    var infiles = 0, outfiles = 0, done = false,
        busboy = new Busboy({ headers: req.headers });

busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
  console.log('File [' + fieldname +']: filename: ' + filename + ', encoding: ' + encoding);
  file.on('data', function(data) {
    console.log('File [' + fieldname +'] got ' + data.length + ' bytes');
  });
  file.on('end', function() {
    console.log('File [' + fieldname +'] Finished');
  });
});
busboy.on('field', function(fieldname, val, valTruncated, keyTruncated) {
  console.log('Field [' + fieldname + ']: value: ' + val);
});
busboy.on('end', function() {
  console.log('Done parsing form!');
  next();
});

req.pipe(busboy);
}

And I'm using it in some express route like so:

var multipartParser= require('./multipartParser');

...

app.post('/enc', multipartParser, function (req, res) {

...

}

and this is one example model:

POST http://localhost:8080/enc HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
Referer: http://localhost:8080/enc
Origin: http://localhost:8080
User-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36
Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryJyMiUl6idThktREs


------WebKitFormBoundaryJyMiUl6idThktREs
Content-Disposition: form-data; name="cont"

some random content
------WebKitFormBoundaryJyMiUl6idThktREs
Content-Disposition: form-data; name="pass"

some random pass
------WebKitFormBoundaryJyMiUl6idThktREs
Content-Disposition: form-data; name="bit"

2
------WebKitFormBoundaryJyMiUl6idThktREs-

Google developer tools don't show all headers, this is all that node has:

console.log(req.headers);
console.log(rawData);

{ host: 'localhost:8080',
  connection: 'keep-alive',
  'content-length': '352',
  accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
  origin: 'http://localhost:8080',
  'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/30.0.1599.101 Safari/537.36',
  'content-type': 'multipart/form-data; boundary=----WebKitFormBoundaryTB2MiQ36fnSJlrhY',
  dnt: '1',
  referer: 'http://localhost:8080/enc',
  'accept-encoding': 'gzip,deflate,sdch',
  'accept-language': 'pt-PT,en-GB;q=0.8,pt;q=0.6,en;q=0.4' }
------WebKitFormBoundaryTB2MiQ36fnSJlrhY
Content-Disposition: form-data; name="cont"

some random content
------WebKitFormBoundaryTB2MiQ36fnSJlrhY
Content-Disposition: form-data; name="pass"

some random pass
------WebKitFormBoundaryTB2MiQ36fnSJlrhY
Content-Disposition: form-data; name="bit"

2
------WebKitFormBoundaryTB2MiQ36fnSJlrhY--

this is the form I'm using for testing the submit:

    <form action="" method="POST" enctype="multipart/form-data">
        <div>
        <label for="cont">Content</label>
        <textarea name="cont" id="cont" /></textarea>
        </div>

        <div>
        <label for="pass">Password</label>
        <input type="password" name="pass" id="pass" />
        </div>

        <div>
        <label for="bit">Bits</label>
        <input type="number" name="bit" id="bit" min="1" max="4" value="1" />
        </div>

        <!--div>
        <label for="png">PNG</label>
        <input type="file" name="png" id="png" />
        </div-->

        <div>
        <input type="submit" />
        </div>
    </form>

If you uncomment the PNG part it works, even if you don't submit a file

I have other multipartParser written with multiparty that is working well, but I would prefer using busboy because of the performance considerations.

BTW, I'm using the library installed from the tarball (master branch)

Is there a workaround to add support for content types such as text/csv?

  throw new Error('Unsupported content type: ' + headers['content-type']);
        ^
Error: Unsupported content type: text/csv
    at Busboy.parseHeaders (/hidden/node_modules/busboy/lib/main.js:66:9)
    at new Busboy (/hidden/node_modules/busboy/lib/main.js:21:10)
    at Server.<anonymous> (/hidden/test-upload.js:11:16)
    at Server.EventEmitter.emit (events.js:98:17)
    at HTTPParser.parser.onIncoming (http.js:2108:12)
    at HTTPParser.parserOnHeadersComplete [as onHeadersComplete] (http.js:121:23)
    at Socket.socket.ondata (http.js:1966:22)
    at TCP.onread (net.js:527:27)

Hi,

I am successfully able to use read POST data on request-received.
But, when I put file-read event after checking/reading field value then, request hang.

Can you tell me a way from your code. How can we register file-read-event after checking the field value from database.
basically a serial request to read field 1st and then after its say read file-data then, file read event should get executed.

Thanks,

Hello how can I parse multiple files at once?

I'm posting multiple files to my node script but only one file works.

Can I use a for loop or something simmilar with busboy?

Is there any particular reason why busboy does not emit any event when max filesize is reached?
https://github.com/mscdex/busboy/blob/master/lib/types/multipart.js#L186

with the current version, the event finish is always invoked and there is no way to understand if the upload have been truncated or not, unless if i count chunks

thank you

Found a bug in busboy/lib/types/multipart.js:103
If content-type of a part is empty eg:

content-type: \r\n

Then express throws an exception.
Probably it is quite dangerous issue because can be used for attacks.
Issue here:

parsed = parseParams(header['content-type'][0]);
contype = parsed[0].toLowerCase();

When using either IE8 or IE9 or IE11 with Intranet settings (and the default value for the custom security setting "Include local directory path when uploading files to a server"), the full file path of an uploaded file is sent to the server (Ex : C:\my_folder\my_image.png) but with backslashes not escaped.

The problem that this raises is that busboy then parses the name as C:my_foldermy_image.png.

Even though this is a bug on IE's part since it is not doubling its backslashes before sending them, it would really be helpful if busboy handled unescaped single backslashes.

The following two new tests fail for the current implementation of busboy's utils.js :

{ source: 'text/plain; filename="C:\\folder\\test.png"',
    expected: ['text/plain', ['filename', 'C:\\folder\\test.png']],
    what: 'Unescaped backslashes should be considered backslashes'
},
{ source: 'text/plain; filename="John \\"Magic\\" Smith.png"',
    expected: ['text/plain', ['filename', 'John "Magic" Smith.png']],
    what: 'Escaped double-quotes should be considered double-quotes'
},

A possible solution to a very similar problem appears to have been addressed here : rack/rack#323 (comment)

Hello -

So you just recommended this library to me over on Stack Exchange. Thought I'd take it for a spin. Problem is, it doesn't seem to be working for me. The page simply hangs after the image is submitted. I'm testing with a very small file -- 321 bytes -- so size can't be the problem. And none of the log statements I've put in the busboy event listeners ever seem to appear.

Here's my code. I'm not seeing anything wrong with this test case. Are you?

var http = require('http'),
    Busboy = require('busboy');

var server = http.createServer(function(request, response) {
    if (request.method === "GET") {
        // on GET request, output web page with a form
        var mypage = '<!doctype html><html><head><meta charset="utf-8">' + 
                        '<title>Submit POST Form</title></head>\r\n<body>' +
                        '<form action="http://127.0.0.1:8008" method="POST" ' + 
                        'enctype="multipart/form-data"> <input name="upload" ' + 
                        'type="file"><p><button type="submit">Submit</button>' + 
                        '</p></form></body></html>\r\n';
        response.writeHead(200, {
            "Content-Type": "text/html",
            "Content-Length": mypage.length
        });
        response.end(mypage);

    } else if (request.method === "POST") {
        console.log('post received');
        var busboy = new Busboy({headers: request.headers}),
            upload = new Buffer([]),
            mimeType = null;
        busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
            mimeType = mimetype;
            console.log('mimetype set to ' + mimeType);
            file.on('data', function(data) {
                console.log('got ' + data.length + ' of data');
                upload = Buffer.concat(upload, data);
            });
            file.on('end', function() {
                console.log("file upload complete");
            });
        });
        busboy.on('finish', function() {
            console.log("form complete");
            response.writeHead(200, {
                "Content-Type": mimeType
            });
            response.end(upload);
        });
    }
});

server.listen(8008);
console.log("server running on port 8008");

[00:06:59 Worker 22753 Critical] TypeError: Object video/, has no method 'toLowerCase' TypeError: Object video/, has no method 'toLowerCase'
    at PartStream.<anonymous> (/var/volafile/app/node_modules/busboy/lib/types/multipart.js:126:31)
    at PartStream.emit (events.js:95:17)
    at HeaderParser.<anonymous> (/var/volafile/app/node_modules/busboy/node_modules/dicer/lib/Dicer.js:51:16)
    at HeaderParser.emit (events.js:95:17)
    at HeaderParser._finish (/var/volafile/app/node_modules/busboy/node_modules/dicer/lib/HeaderParser.js:70:8)
    at SBMH.<anonymous> (/var/volafile/app/node_modules/busboy/node_modules/dicer/lib/HeaderParser.js:42:12)
    at SBMH.emit (events.js:106:17)
    at SBMH._sbmh_feed (/var/volafile/app/node_modules/busboy/node_modules/dicer/node_modules/streamsearch/lib/sbmh.js:159:14)
    at SBMH.push (/var/volafile/app/node_modules/busboy/node_modules/dicer/node_modules/streamsearch/lib/sbmh.js:56:14)
    at HeaderParser.push (/var/volafile/app/node_modules/busboy/node_modules/dicer/lib/HeaderParser.js:48:19)

I just spotted a few of these messages in my logs.

Hello
If I set var busboy = new Busboy({ headers: req.headers, limits: {fields : 3 });
and has more than 3 non-file fields in my post form, the browser will just show "waiting for local host...." and then finally time out showing: "no data received"

cojs/busboy#9

not sure if you want to do this. up to you. i haven't verified what happens myself, but it's better than this guy's report that it just hangs.

I seem to have an issue with Transfer-Encoding: chunked uploads - the file is not fully streamed to disk, even though the browser sent all the data. In fact, it only writes out the first chunk, and treats each additional chunk as a new transfer (without writing it to disk).

Does busboy usually handle this scenario, or do I have to tweak my code for it?

i notice you emit a limit event, but IMO you should also emit a close event both on that specific file stream as well as the busboy parser stream to signify the parser has stopped. i use the close event to properly cleanup stuff.

http://nodejs.org/api/stream.html#stream_event_close

the close event should be emitted after the limit event.

Your documentation says:

Note: if you listen for this event, you should always handle the stream no matter if
you care about the file contents or not (e.g. you can simply just do stream.resume();
if you want to discard the contents), otherwise the 'finish' event will never fire
on the Busboy instance. However, if you don't care about any incoming files,
you can simply not listen for the 'file' event at all and any/all files will be automatically
and safely discarded (these discarded files do still count towards files and parts limits).

However i have seen that even if you don't attach a listener for 'file' event, the file stream remains paused and the 'finish' event is never called.

Tried busboy on a site that has a lot of file uploads. Some requests make it crash (may be 1 in 1000 requests). I think busboy fails to parse the boundary. I currently don't have example data for a request that makes it crash but i'm working on it.

/home/app/server/node_modules/busboy/node_modules/dicer/lib/Dicer.js:74
  this._bparser.push(data);
                ^
TypeError: Cannot call method 'push' of undefined
    at Dicer._write (/home/app/server/node_modules/busboy/node_modules/dicer/lib/Dicer.js:74:17)
    at doWrite (_stream_writable.js:221:10)
    at writeOrBuffer (_stream_writable.js:211:5)
    at Dicer.Writable.write (_stream_writable.js:180:11)
    at Multipart.write (/home/app/server/node_modules/busboy/lib/types/multipart.js:199:24)
    at Busboy._write (/home/app/server/node_modules/busboy/lib/main.js:60:16)
    at doWrite (_stream_writable.js:221:10)
    at writeOrBuffer (_stream_writable.js:211:5)
    at Busboy.Writable.write (_stream_writable.js:180:11)
    at write (_stream_readable.js:583:24)
    at flow (_stream_readable.js:592:7)
    at IncomingMessage.pipeOnReadable (_stream_readable.js:624:5)
    at IncomingMessage.EventEmitter.emit (events.js:92:17)
    at emitReadable_ (_stream_readable.js:408:10)

Hey guys, first of all, great module.

But I am having a problem when I try to upload a json file (multipart) (node v0.10.15) and pipe it to another file. I wanted to pipe to a request, but I got weird errors when server tried to parse the json: basically, it looks like the library is not parsing the entire json file, but only a few lines of it. (I was able to verify by checking the output file)

I checked if the truncate variable was true, but was false. I increased all the limits, but it shouldnt matter, since the json file is really small. Tried with different json files as well.

The library worked great with images/pdfs that I used (different sizes ...), with the same client side fineuploader plugin.

The encoding detected was 7bit, but that was the same in all uploads (of different file types).

Tomorrow I will try to take a look and see if I can find out more about this, but, do you guys have any suggestions that I can try?

My code is exactly like the example, there is nothing more to it.

Thanks for the attention.

Hey,

I'm trying to test the multipart example that writes files to disk with no modification, but it seems to me that the busboy.on('end') is not getting called for files that have more than ~ 100 KB.

For thiny files it seems to work flawlessly

EDIT: I'm testing uploading files in chrome's extension Postman restful client

This is likely not an issue with Busboy, but is it possible?

For example, I want to verify a request before doing something expensive with the file stream.

This is pertaining to a negative use case which I have been trying to debug. I would expect busboy to throw an error somewhere.

I am able to get Postman to send a document as part of a multipart POST request.
Busboy is able to parse out the file, and pass off the file stream. This file stream does not emit the end event, causing my code to hang. And as a consequence busboy does not emit a 'finish' event either.

Just wondering if this really is a bug, I haven't been able to find any informaiton on this anywhere.

Thanks.

https://github.com/visionmedia/supertest

code:

  exports.file = function(req, res) {
    var busboy = new Busboy({ headers: req.headers });

    busboy.on('file', function(field, file, filename, encoding) {
      console.log('file');
    });

    busboy.on('field', function(field, val, valTruncated, keyTruncated) {
      console.log('field');
    });

    var end = 0;
    busboy.on('end', function() {
      end++;
      console.log('end: ' + end);
    });

    req.pipe(busboy);
  };

output:

file
end: 1
field
field
end: 2

If I use busboy.once('end') (as in docs), it ends before I get the fields.

How can I know that parsing both fields and files is done using busboy?

Hello, i'm having problems to use busboy in different express middlewares that handles the same request:

function midleware1(req, res, next){
    var option = {
        headers: req.headers
    };
    var busboy = new Busboy(option);

    busboy.on('field', function(fieldname, val) {
       // some code... needed for a specific validation
    });

    busboy.on('finish', function() {
        next();
    });

    return req.pipe(busboy);
}

function midleware2(req, res, next){
    var option = {
        headers: req.headers
    };
    var busboy = new Busboy(option);

   busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
       // some other code that handle the file upload...
    });

    busboy.on('finish', function() {
        next();
    });

    return req.pipe(busboy);
}

and if i try to use the this two middleware to handle the same request

   app.post("/url",midleware1,midleware1);

Unfortunately it just hangs when /url is requested.

I need to split busboy in two separate middlewares, because i want first to check some params, and only if certain requirements are met, allow the file upload

I have tried to do also req.unpipe(busy) in midleware1, but it does not work.
Can you please help me understand how this can be accomplished with busboy?

Hey there. I'm really glad I'm using this - makes life easy!

I have a tricky problem and was wondering if you have any suggestions, I'm using busboy to parse the form submission from a mailgun webhook. Using the following code (in coffeescript) my problem is that the authentication is sent as fields in the form so I can't actually validate the sender until then, but this logic path would still allow Bad-Guys to spam attachments up to my server, even if I remove them after they aren't validated it would leave a hole. I thought about just saving the file in an array and then write to disk later, but I'm not certain about the memory performance of that - I may have the same vulnerability.

I wasn't quite sure whether this was just a general node.js performance issue or specific to busboy so I thought I'd ask here first as I'm just starting out with busboy. Any help would be greatly appreciated but if you think it's not relevant I'll go ask on SO.

busboy = new Busboy(headers: req.headers)

    preEmail = {}
    attachments = {}

    busboy.on "file", (fieldname, file, filename, encoding, mimetype) ->
        _documentName = "document_#{_attachdata.id}_001#{path.extname filename}"
        _filepath = path.join './uploads/files/emailattachments/', _documentName
        attachments[filename] = _filepath
        file.pipe fs.createWriteStream 

    busboy.on "field", (fieldname, val, fieldnameTruncated, valTruncated) -> preEmail[fieldname] = val

    busboy.on "finish", ->
        # Have to Do Auth Here
        return
    req.pipe busboy

I'm trying to gracefully handle any (user-)aborted uploads and cleanup tmp files - however, I cannot seem to get any event (busboy error, file error) to fire when I end a request during a transfer (tested with Chrome and cUrl). Is there any event I can look for?

I'm guessing that this is actually a dicer bug, but I'm seeing this in my logs (causing server restarts):

  TypeError: Cannot call method 'push' of undefined
      at Dicer._write (/home/app/dojo/node_modules/busboy/node_modules/dicer/lib/Dicer.js:103:17)
      at doWrite (_stream_writable.js:221:10)
      at writeOrBuffer (_stream_writable.js:211:5)
      at Dicer.Writable.write (_stream_writable.js:180:11)
      at Multipart.write (/home/app/dojo/node_modules/busboy/lib/types/multipart.js:227:24)
      at Busboy._write (/home/app/dojo/node_modules/busboy/lib/main.js:67:16)
      at doWrite (_stream_writable.js:221:10)
      at writeOrBuffer (_stream_writable.js:211:5)
      at Busboy.Writable.write (_stream_writable.js:180:11)
      at write (_stream_readable.js:583:24)
      at flow (_stream_readable.js:592:7)
      at _stream_readable.js:560:7
      at process._tickDomainCallback (node.js:459:13)

I can't reproduce this in tests though.

Here are the headers:

{ host: '[redacted]',
    accept: '*/*',
    'accept-encoding': 'gzip, deflate',
    'accept-language': 'en-us',
    'content-type': 'multipart/form-data; charset=utf-8; boundary=0xKhTmLbOuNdArY',
    cookie: '[redacted]',
    'user-agent': '[redacted]/122 CFNetwork/672.0.8 Darwin/14.0.0',
    'x-client-build': '122',
    'x-client-identifier': 'iOS',
    'x-client-version': '2.4',
    'x-forwarded-for': '[redacted]',
    'x-forwarded-port': '80',
    'x-forwarded-proto': 'http',
    'content-length': '1449',
    connection: 'close' }

These don't seem to be sufficient to reproduce the error in tests though. I don't have an example body either, unfortunately.

Using busboy 0.2.1.

Thanks!

Steps to reproduce:

1. Install and start the server

   • mkdir test
   • cd test
   • npm install busboy
   • vim server.js <= paste the example from the readme.md
   • node server.js

2. Perform an upload using a file with ASCII characters

curl -v -F 'id=123' -F "[email protected];type=video/mp4" http://localhost:8000

In console, you get this:

File [file]: filename: test.mp4, encoding: 7bit
File [file] got 65305 bytes
Field [id]: value: '123'
File [file] got 65536 bytes
[...]
File [file] Finished
Done parsing form!

Curl call finishes correctly:

< HTTP/1.1 100 Continue
< HTTP/1.1 303 See Other
< Connection: close
< Location: /
< Date: Tue, 10 Dec 2013 09:21:15 GMT
< Transfer-Encoding: chunked

1. Perform an upload using a file with UTF-8 characters

cp test.mp4 お疲れ様です.mp4
curl -v -F 'id=123' -F "file=@お疲れ様です.mp4;type=video/mp4" http://localhost:8000

In console, you get this:

Field [id]: value: '123'

Curl call will eventually timeout:

< HTTP/1.1 100 Continue

• Recv failure: Connection reset by peer
• Closing connection 0

I noticed this while testing today. If a request comes in with a content-length of 0 (according to req.headers) then the finish method is never called.

I think this is an issue because the finish method should always be called, even if there is no data.

I tested this with the example in the first example in the readme and Busboy 0.2.3.

just pulling the TODO from the code out into a github issue for visibility

busboy emits end at the end of the parsing of parts, but nothing at the end of the processing of parts. Therefore it is not possible to trigger a cleanup/collection after the end of the processing.

As a result, the code provided by senchalabs/connect#898 (comment) cannot be chained with a post-processing step that deals with every parts at a time.

I want to create my own DIR.
I used mkdirp it saved file but file size is 0 byte
What should I do?

busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
            var saveTo = '/uploads/date/time/file.jpg';
        mkdirp(saveTo, function(err) {
        if (err) {
            console.error(err);
        } else {
            console.log('Save File..');
            file.pipe(fs.createWriteStream(saveTo));
        }
    });
});

Just to say thanks, this module saved me a lot of time today =)

Hey, I was debugging an issue which was caused by uploading ~750 Files at once, and noticed that for some strange reason, I wasn't getting any more "file" events on seemingly random times during the transfer, and the upload would freeze in turn. I was uploading the same folder (recursively) over and over, and on rare occasion, it even finished.

For now, I've increased the fileHwm to 1Mb which seemed to have eliminated the issue, but I'm not sure, if that's the right approach to this, or if I may have uncovered a bug in this module.

Also to note, the issue really looks like it's related to the amount of files here, as I can upload a 10GB single file just fine.

Hi! First of all thanks for the great module!

I've encountered the following problem - it seems that it doesn't detect mime types right. At least when I'm posting image files it says that the mime type is "text/plain".

If file is not known file type, it will be octet stream, on event 'file' there is no way to cancel that file downloading.

So the idea is to allow logic to cancel specific files to be downloaded, that means I want to prevent upload from browser in first place.

The only way found so far, is to force close request socket that leads to 502 (browser thinks so) while app is still running.

How to cancel download on the server and throw some status code without downloading whole file from client?

Great library, thanks for all the work maintaining it!

Busboy tests all pass for me on node 0.10.28, but after upgrading to 0.10.29, I see:

ploer-macbook:busboy ploer$ npm test

> [email protected] test /Users/ploer/sandbox/butter/busboy
> node test/test.js

-
assert.js:92
  throw new assert.AssertionError({
        ^
AssertionError: [test-types-multipart/Fields and files]: Result mismatch:
Parsed: [ 'field', 'file_name_0', '', false, false ]
Expected: [ 'field', 'file_name_0', 'super alpha file', false, false ]
    at /Users/ploer/sandbox/butter/busboy/test/test-types-multipart.js:224:14
    at Array.forEach (native)
    at Busboy.<anonymous> (/Users/ploer/sandbox/butter/busboy/test/test-types-multipart.js:223:13)
    at Busboy.emit (events.js:92:17)
    at Busboy.emit (/Users/ploer/sandbox/butter/busboy/lib/main.js:31:35)
    at /Users/ploer/sandbox/butter/busboy/lib/types/multipart.js:52:13
    at process._tickCallback (node.js:419:13)
    at Function.Module.runMain (module.js:499:11)
    at startup (node.js:119:16)
    at node.js:906:3
npm ERR! Test failed.  See above for more details.
npm ERR! not ok code 0

It looks to me like the onData at multipart.js:197 is getting called after the onEnd at mulitpart.js:209 for a given field, but I don't really have any idea why that might be.

Hello,
when I try to upload files using ajax the content type will always be
application/octet-stream even if I set it to multipart/form-data.

Thats my current approach:
http://www.filecatalyst.com/xhr2-uploads-intro

How to properly upload files using ajax , how to force application/octet-streamfor?

This is not enough:

<form enctype="multipart/form-data" id="uploadForm">
select file(s): <input name="files" type="file" multiple="multiple" id="files"/>
<input id="uploadFiles" type="button" value="Upload File" />
</form>

Use this payload: http://superjoe.s3.amazonaws.com/temp/busboy-payload.pdf

node version 0.10.24
busboy 0.0.13

var http = require('http');
var Busboy = require('busboy');

var server = http.createServer(function(req, res) {
  if (req.url === '/') {
    res.statusCode = 200;
    res.end('<!doctype html>'+
      '<html>'+
      '<head>'+
      '</head>'+
      '<body>'+
      '<form action="/upload" method="post" enctype="multipart/form-data">'+
      '  <input type="file" name="avatar">'+
      '  <input type="submit">'+
      '</form>'+
      '</body>'+
      '</html>');
  } else if (req.url === '/upload') {
    var bboy = new Busboy({
      headers : req.headers,
      limits  : {fileSize:2*1024*1024, files:1}
    });

    bboy.on('file', function(fieldname, file, filename, encoding, mimetype){
      console.log("aborting the request");
      res.statusCode = 400;
      res.end("hello browser");
    });
    req.pipe(bboy);
  } else {
    console.info("unexpected path", req.url);
  }
});

server.listen(9191, function() {
  console.info("listening at http://0.0.0.0:9191/");
});

When I upload that payload, the request completely hangs. The browser never sees "hello browser".

...also, how to handle multiple file uploads and send one response when complete.

Hi,

I am getting this error. Today I have install:busboy for parsing POST multi-part encoded requested json-data using express4 framework..

I am receiving error when receiving a POST request encoded with multipart .

Hi, my use case is that I'm specifying a particular location where I would like to upload a file. The location is in the same form as the file. In order to do this, I'm using a promise that forces the file event handler to wait on the field event handler. See this code:

console.debug('Responding to POST request to ./views/uploadFile');
var promisedFolder = new Deferred(),
    responseObj = {files:[]};

var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
    console.debug('File [' + fieldname + ']: filename: ' + filename);
    promisedFolder.then(function(folder) {
        console.debug('Got folder from request, writing file');
        var fullPath = path.normalize('/' + folder + '/' + filename);
        file.pipe(fs.createWriteStream(fullPath));
        file.on('error', function(err) {
            console.debug('Got error streaming file');
        });
        responseObj.files.push({name:filename, url:'#'});
    });
});
busboy.on('field', function(fieldname, val, valTruncated, keyTruncated) {
    console.debug('Field [' + fieldname + ']: value: ' + val);
    promisedFolder.resolve(val);
});
busboy.on('finish', function() {
    console.debug('Done parsing form!');
    res.send(responseObj);
});

req.pipe(busboy);

This is what I get in the logs, but then nothing else happens.

DEBUG: 2014-03-13T16:14:43.212Z: File [files[]]: filename: jare-2008-articlespdf.pdf
DEBUG: 2014-03-13T16:14:43.212Z: Field [path]: value: /data/Bills/
DEBUG: 2014-03-13T16:14:43.212Z: Got folder from request, writing file

Weirdly, If I resend the exact same request, the upload goes through.

Thoughts?

It would be useful if you expose the content-type of the multipart file as event-param:

multipart.js:96

        boy.emit('file', fieldname, file, filename, encoding, contype);

I wanted to allow no files uploaded so I set limits : { files : 0 }} but since busboy does limits.files || Infinity it defaults to infinity when the limit is 0. This is the cases with a few other configuration parameters too.