From a23ad75c2c53a211948eac6f8d8bb30e17b1e0db Mon Sep 17 00:00:00 2001
From: mhassman <mark@hassman.org>
Date: Thu, 21 Jul 2016 12:54:45 -0400
Subject: [PATCH 1/6] request_processor.lua:   - require server-generated
 session_id value.

index.html:
  - removed client generation of session_id value.
  - client now calculates crc values for chunks.. sends to server for verification.
---
 example/html/index.html | 86 +++++++++++++++++++++++++++++++++++++----
 request_processor.lua   | 14 ++++++-
 2 files changed, 91 insertions(+), 9 deletions(-)

diff --git a/example/html/index.html b/example/html/index.html
index d58070a..ede00a2 100644
--- a/example/html/index.html
+++ b/example/html/index.html
@@ -22,14 +22,17 @@ <h1>Example Chunked Uploader</h1>
 
 <script>
 
-function doUpload(url, extraParams, sessionID, file, progress, success) {
+function doUpload(url, extraParams, file, progress, success) {
     var chunkSize = 32000,// this is first chunk size, which will be adaptively expanded depending on upload speed
         aborting = false,
         TO = null,
         lastChecksum = null,
         offset = 0,
-        lastChunkTime = 4000; //used for adaptive chunk size recalculation
+        lastChunkTime = 4000, //used for adaptive chunk size recalculation
+        calculateCRCs = true,
+        crcBlockSize = 4096;
     var xhr;
+    var sessionID;
 
     function adjustChunkSize(startTime) {
         var end = new Date().getTime();
@@ -48,6 +51,36 @@ <h1>Example Chunked Uploader</h1>
         }
     }
 
+	var crc32 = (function() {
+
+		var table = new Uint32Array(256);
+
+		// Pre-generate crc32 polynomial lookup table
+		// http://wiki.osdev.org/CRC32#Building_the_Lookup_Table
+		// ... Actually use Alex's because it generates the correct bit order
+		//     so no need for the reversal function
+		for (var i=256; i--;) {
+			var tmp = i;
+			for(var k=8; k--;)
+			{
+				tmp = tmp & 1 ? 3988292384 ^ tmp >>> 1 : tmp >>> 1;
+			}
+			table[i] = tmp;
+		}
+
+		// crc32b
+		// Example input        : [97, 98, 99, 100, 101] (Uint8Array)
+		// Example output       : 2240272485 (Uint32)
+		return function (data) {
+			var crc = -1; // Begin with all bits set ( 0xffffffff )
+			for(var i=0, l=data.length; i<l; i++) {
+				crc = crc >>> 8 ^ table[ crc & 255 ^ data[i] ];
+			}
+			return (crc ^ -1) >>> 0; // Apply binary NOT
+		};
+
+	})();
+
     var uploadNextChunk = function() {
             TO = null;
             var chunkStartTime = new Date().getTime();
@@ -88,6 +121,7 @@ <h1>Example Chunked Uploader</h1>
                 }
 
                 if (xhr.readyState >= 4) {
+
                     if (xhr.status === 200) {
                         progress((chunkEnd + 1) / file.size);
 
@@ -95,6 +129,7 @@ <h1>Example Chunked Uploader</h1>
                         success(xhr.responseText);
 
                     } else if (xhr.status === 201) {
+	                    sessionID = xhr.getResponseHeader('X-Session-Id');
                         offset = chunkEnd + 1;
                         progress(offset / file.size);
                         adjustChunkSize(chunkStartTime);
@@ -130,13 +165,25 @@ <h1>Example Chunked Uploader</h1>
             xhr.setRequestHeader('Content-Type', 'application/octet-stream');
             xhr.setRequestHeader('Content-Disposition', 'attachment; filename="' + encodeURIComponent(file.name) + '"');
             xhr.setRequestHeader('X-Content-Range', 'bytes ' + chunkStart + '-' + chunkEnd + '/' + file.size);
-            xhr.setRequestHeader('X-Session-ID', sessionID);
+            if (sessionID != null) {
+                xhr.setRequestHeader('X-Session-ID', sessionID);
+            }
             if(lastChecksum) {
                 //client must pass the checksum of all previous chunks
                 //this way server will continue checksum calculation
                 xhr.setRequestHeader('X-Last-Checksum', lastChecksum);
             }
-            xhr.send(currentBlob);
+            if (calculateCRCs) {
+				blobToArrayBuffer(currentBlob, function (arrayBuffer) {
+					var byteArray = new Uint8Array(arrayBuffer);
+					var crc = createCRC(byteArray);
+					xhr.setRequestHeader('X-Checksum', crc.toString(16));
+					//console.log('X-Checksum', crc + " " + crc.toString(16));
+					xhr.send(currentBlob);
+				});
+			} else {
+				xhr.send(currentBlob);
+			}
         };
 
     TO = setTimeout(uploadNextChunk, 1);
@@ -161,6 +208,32 @@ <h1>Example Chunked Uploader</h1>
             uploadNextChunk();
         }
     };
+
+	function blobToArrayBuffer (blob, callback) {
+		var fileReader     = new FileReader();
+		fileReader.onload  = function() {
+			callback(this.result);
+		};
+		fileReader.readAsArrayBuffer(blob)
+	}
+
+	function createCRC (byteArray) {
+		var crcBlockCount = Math.ceil(byteArray.length / crcBlockSize);
+		var crc = 0;
+		var crcBlockStart = 0;
+		for (var i=0; i<crcBlockCount; i++) {
+			var crcBlockEnd = crcBlockStart + crcBlockSize;
+			if (crcBlockEnd > byteArray.length) {
+				crcBlockEnd = byteArray.length;
+			}
+			var crcBlock = byteArray.subarray(crcBlockStart, crcBlockEnd);
+			crc = crc32(crcBlock);
+			//console.log("crc's: " + i + ": " + crc + " crcBlock.length: " + crcBlock.length);
+			crcBlockStart = crcBlockEnd;
+		}
+		return crc;
+	}
+
 }
 
 
@@ -183,10 +256,7 @@ <h1>Example Chunked Uploader</h1>
 
         var file = document.getElementById('file-to-upload').files[0];
 
-        //generate random long number for SessionID
-        var sessionID = Math.round(Math.pow(10,17)*Math.random());
-
-        doUpload('/upload', 'test=1', sessionID, file, function(progress) {
+        doUpload('/upload', 'test=1', file, function(progress) {
 
             console.log('Total file progress is ' + Math.floor(progress * 100) + '%');
 
diff --git a/request_processor.lua b/request_processor.lua
index 810e3af..3b80c56 100644
--- a/request_processor.lua
+++ b/request_processor.lua
@@ -19,8 +19,13 @@ local util = require('util')
 
 module(...)
 
+
+
+
+
 local mt = { __index = _M }
 
+
 _M.chunk_size = 4096
 _M.socket_timeout = 30000
 
@@ -44,6 +49,7 @@ end
 
 -- Checks request headers and creates upload context instance
 function new(self, handlers)
+
     local ctx = {}
     local headers = ngx.req.get_headers()
 
@@ -77,15 +83,21 @@ function new(self, handlers)
     end
 
     local session_id = headers["session-id"] or headers["x-session-id"]
+    if ctx.first_chunk and session_id then
+        ngx.log(ngx.ERR, "Client-generated session-id is not permitted.")
+        return nil, {412, "Client-generated session-id is not permitted." }
+    end
     if not session_id then
         if not ctx.first_chunk then
+            ngx.log(ngx.ERR, "Session-id is required for chunked upload.")
             return nil, {412, "Session-id is required for chunked upload." }
         else
             session_id = util.random_sha1()
         end
     else
         if session_id:match('%W') then
-            return nil, {412, string.format("Session-id is invalid only alphanumeric value are accepted, was %s", session_id)}
+            ngx.log(ngx.ERR, string.format("Session-id contains invalid characters. Only alphanumeric value are accepted, was %s", session_id))
+            return nil, {412, "Session-id contains invalid characters." }
         end
     end
 

From 220fc5cab0f0bb05af54320e6aca03720c69d94d Mon Sep 17 00:00:00 2001
From: mhassman <mark@hassman.org>
Date: Sun, 24 Jul 2016 09:43:31 -0400
Subject: [PATCH 2/6] - removed dependency on ffi - for those of us who don't
 run openresty. - log error on chunk crc mismatch.

---
 crc32.lua | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/crc32.lua b/crc32.lua
index c6aa077..985013e 100644
--- a/crc32.lua
+++ b/crc32.lua
@@ -2,21 +2,20 @@
 
 -- CRC32 checksum
 
-local ffi = require('ffi')
 local tonumber = tonumber
 local string = string
 local ngx = ngx
 local table = table
 
+
+local zlib = require ('zlib')
+
 module(...)
 
-local zlib = ffi.load('z')
-ffi.cdef[[
-    unsigned long crc32(unsigned long crc, const char *buf, unsigned len );
-]]
 
 function crc32(data, lastnum)
-  return tonumber(zlib.crc32(lastnum, data, #data))
+  return zlib.crc32()(data, lastnum)
+  -- return zlib.crc32()(lastnum, data, #data)
 end
 
 function validhex(crchex) return #crchex <= 8 and string.match(crchex, "^%x+$") end
@@ -32,6 +31,7 @@ function handler()
   return {
     on_body_start = function (self, ctx)
       ctx.current_checksum = ctx.last_checksum and tonumber(ctx.last_checksum, 16) or ( ctx.first_chunk and 0 )
+
       -- stop checksum processing if X-Last-Checksum is not present for non first chunk
       if not ctx.current_checksum then
         self.on_body = nil
@@ -40,12 +40,13 @@ function handler()
     end,
 
     on_body = function (self, ctx, body)
-      ctx.current_checksum = crc32(body, ctx.current_checksum)
+        ctx.current_checksum = crc32(body, ctx.current_checksum)
     end,
 
     on_body_end = function (self, ctx)
       if ctx.checksum then
         if tonumber(ctx.checksum,16) ~= ctx.current_checksum then
+          ngx.log(ngx.ERR, string.format("Chunk checksum mismatch client=[%s] server=[%s]", ctx.checksum, tohex(ctx.current_checksum)))
           return {400, string.format("Chunk checksum mismatch client=[%s] server=[%s]", ctx.checksum, tohex(ctx.current_checksum))}
         end
       else

From f66146883d96ff6c505146a3de5c6c92b3ab17d3 Mon Sep 17 00:00:00 2001
From: mhassman <mark@hassman.org>
Date: Sun, 24 Jul 2016 09:56:20 -0400
Subject: [PATCH 3/6] - added mime type content checking for image files.

---
 big-upload.lua |  9 +++++--
 mime.lua       | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 mime.lua

diff --git a/big-upload.lua b/big-upload.lua
index 38afb9f..6f9f8d7 100644
--- a/big-upload.lua
+++ b/big-upload.lua
@@ -4,7 +4,8 @@
 local config = {
     package_path = ngx.var.package_path,
     bu_checksum = ('on' ==  ngx.var.bu_checksum),
-    bu_sha1 = ('on' == ngx.var.bu_sha1)
+    bu_sha1 = ('on' == ngx.var.bu_sha1),
+    bu_mime = ('on' == ngx.var.bu_mime)
 }
 
 if config.package_path then
@@ -15,6 +16,7 @@ local file_storage_handler = require "file_storage_handler"
 local backend_file_storage_handler = require "backend_file_storage_handler"
 local crc32 = require('crc32')
 local sha1= require('sha1_handler')
+local mime = require('mime')
 
 local function report_result(info)
   if type(info) == "table" then
@@ -62,7 +64,10 @@ if config.bu_checksum then
   table.insert(handlers, crc32.handler())
 end
 if config.bu_sha1 then
- table.insert(handlers, sha1.handler(ngx.var.file_storage_path))
+  table.insert(handlers, sha1.handler(ngx.var.file_storage_path))
+end
+if config.bu_mime then
+  table.insert(handlers, mime.handler())
 end
 table.insert(handlers, storage_handler)
 
diff --git a/mime.lua b/mime.lua
new file mode 100644
index 0000000..7cbd53b
--- /dev/null
+++ b/mime.lua
@@ -0,0 +1,67 @@
+-- this file analyzes the first chunk for recognizable mime types
+-- it depends on ctx.authorized_mime_type value added during authentication
+
+local ngx = ngx
+local string = string
+
+
+module(...)
+
+
+
+function find_mime_type (bytes)
+  local b = ""
+  for i=1, 8 do
+    b = b .. string.upper(string.format("%x", string.byte(bytes, i)))
+    if b     == "FFD8FFDB" then return "jpg"
+    elseif b == "FFD8FFE0" then return "jpg"
+    elseif b == "49460001" then return "jpg"
+    elseif b == "FFD8FFE1" then return "jpg"
+    elseif b == "69660000" then return "jpg"
+    elseif b == "49492A00" then return "tif"
+    elseif b == "4D4D002A" then return "tif"
+    end
+    if b     == "474946383761" then return "gif"
+    elseif b == "474946383961" then return "gif"
+    elseif b == "89504E470D0A" then return "gif"
+    end
+    if b == "89504E470D0A1A0A" then return "png"
+    end
+  end
+  return nil
+end
+
+
+function handler()
+  return {
+
+    on_body_start = function (self, ctx)
+    end,
+
+    on_body = function (self, ctx, body)
+      if ctx.first_chunk then
+        if ngx.ctx.authorized_mime_type == nil then
+          self.on_body_end = nil
+          ngx.log(ngx.ERR, "Authorized mime type is not defined")
+          return {400, "Authorized mime type is not defined"}
+        end
+        if ctx.detected_mime_type ~= nil then  -- already detected...
+          return
+        end
+        ctx.detected_mime_type = find_mime_type(body)
+        if ngx.ctx.authorized_mime_type ~= ctx.detected_mime_type then
+          self.on_body_end = nil
+          if ctx.detected_mime_type == nil then
+            ctx.detected_mime_type = ""
+          end
+          ngx.log(ngx.ERR, string.format("Authorized mime type mismatch. Authorized: [%s]. Detected: [%s]", ngx.ctx.authorized_mime_type, ctx.detected_mime_type))
+          return {400, string.format("Authorized mime type mismatch. Authorized: [%s]. Detected: [%s]", ngx.ctx.authorized_mime_type, ctx.detected_mime_type)}
+        end
+      end
+    end,
+
+    on_body_end = function (self, ctx)
+    end
+
+  }
+end

From 655a75011939ccadfb2675855a97316241e63f5b Mon Sep 17 00:00:00 2001
From: mhassman <mark@hassman.org>
Date: Mon, 25 Jul 2016 10:46:11 -0400
Subject: [PATCH 4/6] - optionally, use filename specified from upstream lua
 processor (ngx.ctx.file_name).

- optionally, error if file length doesn't match breadcrumb specified from upstream lua processor (ngx.ctx.file_length).
---
 backend_file_storage_handler.lua |  4 ++++
 file_storage_handler.lua         |  4 ++++
 request_processor.lua            | 11 +++++++++++
 3 files changed, 19 insertions(+)

diff --git a/backend_file_storage_handler.lua b/backend_file_storage_handler.lua
index 4faefda..2b98d82 100644
--- a/backend_file_storage_handler.lua
+++ b/backend_file_storage_handler.lua
@@ -35,6 +35,10 @@ end
 -- override
 local function on_body_start(self, ctx)
   local file_path = concat({self.dir, ctx.id}, "/")
+  if ctx.file_name ~= nil then
+    -- use explicitly specified filename..
+    file_path = concat({self.dir, ctx.file_name}, "/")
+  end
   ctx.file_path = file_path
   return self:init_file(ctx)
 end
diff --git a/file_storage_handler.lua b/file_storage_handler.lua
index e616b58..224953c 100644
--- a/file_storage_handler.lua
+++ b/file_storage_handler.lua
@@ -58,6 +58,10 @@ end
 
 local function on_body_start(self, ctx)
   ctx.file_path = concat({self.dir, ctx.id}, "/")
+  if ctx.file_name ~= nil then
+    -- use explicitly specified filename..
+    file_path = concat({self.dir, ctx.file_name}, "/")
+  end
   return self:init_file(ctx)
 end
 
diff --git a/request_processor.lua b/request_processor.lua
index 3b80c56..b69ed38 100644
--- a/request_processor.lua
+++ b/request_processor.lua
@@ -82,6 +82,12 @@ function new(self, handlers)
       ctx.first_chunk = true
     end
 
+    -- optionally, verify total file size against breadcrumb left from lua-auth script..
+    if ngx.ctx.file_length ~= range_total then
+        ngx.log(ngx.ERR, string.format("Invalid size. Authorized: [%s], but specified: [%s]", ngx.ctx.file_length, range_total))
+        return nil, {412, string.format("Invalid size. Authorized: [%s], but specified: [%s]", ngx.ctx.file_length, range_total)}
+    end
+
     local session_id = headers["session-id"] or headers["x-session-id"]
     if ctx.first_chunk and session_id then
         ngx.log(ngx.ERR, "Client-generated session-id is not permitted.")
@@ -188,6 +194,11 @@ function new(self, handlers)
 
     ctx.id = session_id
 
+    -- optionally use specified filename from breadcrumb left by lua auth script..
+    if ngx.ctx.file_name ~= nil then
+      ctx.file_name = ngx.ctx.file_name
+    end
+
     return setmetatable({
         socket = socket,
         chunk_size = chunk_size,

From acedaaeb79ba594ff25dd9f84b2d4f95ed868875 Mon Sep 17 00:00:00 2001
From: mhassman <mark@hassman.org>
Date: Tue, 26 Jul 2016 05:58:52 -0400
Subject: [PATCH 5/6] - added file_name to returned data from completed upload.

- added move successful uploads if $success_destination_dir variable is set in .conf.
---
 backend_file_storage_handler.lua | 21 +++++++++++++++++++--
 file_storage_handler.lua         | 17 +++++++++++++++++
 request_processor.lua            |  5 +++++
 3 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/backend_file_storage_handler.lua b/backend_file_storage_handler.lua
index 2b98d82..ad4f4fa 100644
--- a/backend_file_storage_handler.lua
+++ b/backend_file_storage_handler.lua
@@ -10,7 +10,7 @@ local ngx = ngx
 local string = string
 local concat = table.concat
 local error = error
-
+local os = os
 
 module(...)
 
@@ -25,6 +25,7 @@ local function end_backend(self, ctx)
           id = ctx.id,
           path = ctx.file_path,
           name = ctx.get_name(),
+          file_name = ctx.file_name,
           checksum = ctx.checksum,
           sha1 = ctx.sha1
         })
@@ -32,6 +33,15 @@ local function end_backend(self, ctx)
   end
 end
 
+local function move_file(self, ctx)
+  if ctx.file_path and ctx.success_destination_dir then
+    local ret, err = os.rename(ctx.file_path, concat({ctx.success_destination_dir, ctx.file_name}, "/"))
+    if ret == nil or err ~= nil then
+      return string.format("Failed to move completed file: %s to: %s. Error: %s", ctx.file_path, ctx.success_destination_dir, err)
+    end
+  end
+end
+
 -- override
 local function on_body_start(self, ctx)
   local file_path = concat({self.dir, ctx.id}, "/")
@@ -46,7 +56,14 @@ end
 -- override
 local function on_body_end(self, ctx)
   self:close_file()
-  -- call backend if finished
+  if ctx.range_to + 1 == ctx.range_total then
+    local ret = move_file(self, ctx)
+    if ret then
+      ngx.log(ngx.ERR, ret)
+      return ngx.ERROR
+    end
+  end
+  -- call backend if finished & move successful
   local res = end_backend(self, ctx)
   return {201, string.format("0-%d/%d", ctx.range_to, ctx.range_total), response = res }
 end
diff --git a/file_storage_handler.lua b/file_storage_handler.lua
index 224953c..7301493 100644
--- a/file_storage_handler.lua
+++ b/file_storage_handler.lua
@@ -9,6 +9,7 @@
 local setmetatable = setmetatable
 local concat = table.concat
 local io = io
+local os = os
 local string = string
 local error = error
 local ngx = ngx
@@ -56,6 +57,15 @@ local function close_file(self)
   end
 end
 
+local function move_file(self, ctx)
+  if ctx.file_path and ctx.success_destination_dir then
+    local ret, err = os.rename(ctx.file_path, concat({ctx.success_destination_dir, ctx.file_name}, "/"))
+    if ret == nil or err ~= nil then
+      return string.format("Failed to move completed file: %s to: %s. Error: %s", ctx.file_path, ctx.success_destination_dir, err)
+    end
+  end
+end
+
 local function on_body_start(self, ctx)
   ctx.file_path = concat({self.dir, ctx.id}, "/")
   if ctx.file_name ~= nil then
@@ -78,6 +88,13 @@ end
 
 local function on_body_end(self, ctx)
   close_file(self)
+  if ctx.range_to + 1 == ctx.range_total then
+    local ret = move_file(self, ctx)
+    if ret then
+      ngx.log(ngx.ERR, ret)
+      return ngx.ERROR
+    end
+  end
   -- return what what we have on server
   return {201, string.format("0-%d/%d", ctx.range_to, ctx.range_total) }
 end
diff --git a/request_processor.lua b/request_processor.lua
index b69ed38..94e4179 100644
--- a/request_processor.lua
+++ b/request_processor.lua
@@ -199,6 +199,11 @@ function new(self, handlers)
       ctx.file_name = ngx.ctx.file_name
     end
 
+    -- add optional move-to directory for successfully completed uploads..
+    if ngx.var.success_destination_dir then
+      ctx.success_destination_dir = ngx.var.success_destination_dir
+    end
+
     return setmetatable({
         socket = socket,
         chunk_size = chunk_size,

From a38bd83b86d59d49c3a4ac6a88bbe708a9a6b4d9 Mon Sep 17 00:00:00 2001
From: mhassman <mark@hassman.org>
Date: Wed, 27 Jul 2016 12:21:01 -0400
Subject: [PATCH 6/6] - added proxying miscellaneous (ngx.ctx.misc) values from
 the upstream lua modules to backend storage handler data.

---
 backend_file_storage_handler.lua | 33 +++++++++++++++++++++++---------
 request_processor.lua            |  5 +++++
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/backend_file_storage_handler.lua b/backend_file_storage_handler.lua
index ad4f4fa..d838510 100644
--- a/backend_file_storage_handler.lua
+++ b/backend_file_storage_handler.lua
@@ -17,18 +17,33 @@ module(...)
 local function end_backend(self, ctx)
   -- last chunk commited?
   if ctx.range_to + 1 == ctx.range_total then
+
+    -- seed body arguments into table..
+    local body_args = {
+            size = ctx.range_total,
+            id = ctx.id,
+            path = ctx.file_path,
+            name = ctx.get_name(),
+            file_name = ctx.file_name,
+            checksum = ctx.checksum,
+            sha1 = ctx.sha1
+    }
+
+    -- parameterize misc value if populated..
+    -- note: misc must be a string of uri-style parameters. i.e. a=1&b=2
+    -- fucking nginx strips inbound complex key->value tables into simple string arrays
+    if ctx.misc ~= nil then
+      for kv in string.gmatch(ctx.misc, "[^&]+") do
+        for k, v in string.gmatch(kv, "(%w.+)=(%w.+)") do
+          body_args[k] = v
+        end
+      end
+    end
+
     ngx.req.set_header('Content-Type', 'application/x-www-form-urlencoded')
     return ngx.location.capture(self.backend, {
         method = ngx.HTTP_POST,
-        body = ngx.encode_args({
-          size = ctx.range_total,
-          id = ctx.id,
-          path = ctx.file_path,
-          name = ctx.get_name(),
-          file_name = ctx.file_name,
-          checksum = ctx.checksum,
-          sha1 = ctx.sha1
-        })
+        body = ngx.encode_args(body_args)
     })
   end
 end
diff --git a/request_processor.lua b/request_processor.lua
index 94e4179..143ca0a 100644
--- a/request_processor.lua
+++ b/request_processor.lua
@@ -199,6 +199,11 @@ function new(self, handlers)
       ctx.file_name = ngx.ctx.file_name
     end
 
+    -- optionally include misc breadcrumbs left by lua auth script..
+    if ngx.ctx.misc ~= nil then
+      ctx.misc = ngx.ctx.misc
+    end
+
     -- add optional move-to directory for successfully completed uploads..
     if ngx.var.success_destination_dir then
       ctx.success_destination_dir = ngx.var.success_destination_dir