From c005620e84dfb7cb8f5a4c1b59a713af8bfa1111 Mon Sep 17 00:00:00 2001 From: nic Date: Mon, 14 Apr 2025 08:12:27 -0400 Subject: [PATCH] fix: memory limitations due to 32 bit architecture; now using manual form field/file processing; hopefully it continues to work --- internal/handlers/web/documents.go | 197 ++++++++++++++--------------- 1 file changed, 97 insertions(+), 100 deletions(-) diff --git a/internal/handlers/web/documents.go b/internal/handlers/web/documents.go index eea2cf0..5f3c287 100644 --- a/internal/handlers/web/documents.go +++ b/internal/handlers/web/documents.go @@ -6,12 +6,9 @@ import ( "fmt" "io" "log" - "mime/multipart" "net/http" "path/filepath" - "regexp" "sort" - "strconv" "strings" "sync" "time" @@ -197,25 +194,84 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { return } - // Parse the multipart form with a 30MB limit - if err := r.ParseMultipartForm(30 << 20); err != nil { - http.Error(w, fmt.Sprintf("Unable to parse form: %v", err), http.StatusBadRequest) + // Custom multipart form processing for 32-bit systems + reader, err := r.MultipartReader() + if err != nil { + http.Error(w, "Unable to get multipart reader: "+err.Error(), http.StatusBadRequest) return } - // Get the job numbers from either of the possible form fields - jobNumbers := r.FormValue("jobNumbers") + // Store form values and file parts + formValues := make(map[string]string) + + // Read all file contents + type DocumentWithContent struct { + Name string + Type string + FileContent []byte + FormField string // Store the original form field name + } + var docsWithContent []DocumentWithContent + + // First pass: collect all form fields and files + log.Printf("--- Starting multipart form processing ---") + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + log.Printf("Error reading multipart part: %v", err) + break + } + + formName := part.FormName() + fileName := part.FileName() + + // If not a file, it's a regular form value + if fileName == "" { + // Read the form field value + valueBytes, err := io.ReadAll(part) + if err != nil { + log.Printf("Error reading form field %s: %v", formName, err) + continue + } + value := string(valueBytes) + formValues[formName] = value + log.Printf("Form field: %s = %s", formName, value) + } else if strings.HasPrefix(formName, "document-file-") { + // It's a file upload field + // Read file content + fileContent, err := io.ReadAll(part) + if err != nil { + log.Printf("Error reading file content for %s: %v", fileName, err) + continue + } + + log.Printf("Found file: %s (size: %d bytes) in field: %s", + fileName, len(fileContent), formName) + + // Store the file with its original field name for later processing + docsWithContent = append(docsWithContent, DocumentWithContent{ + Name: fileName, // Default to original filename, will be updated with form values + Type: "", // Will be set from form values + FileContent: fileContent, + FormField: formName, + }) + } + } + + // Get job numbers from form values + jobNumbers := formValues["jobNumbers"] if jobNumbers == "" { - jobNumbers = r.FormValue("job-ids") + jobNumbers = formValues["job-ids"] if jobNumbers == "" { - log.Printf("No job numbers provided. Form data: %+v", r.Form) + log.Printf("No job numbers found in form values: %+v", formValues) http.Error(w, "No job numbers provided", http.StatusBadRequest) return } } - // Log the form data for debugging - log.Printf("Form data: %+v", r.Form) log.Printf("Job numbers: %s", jobNumbers) // Split the job numbers @@ -225,110 +281,51 @@ func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) { return } - // Regular expression to match file field patterns - filePattern := regexp.MustCompile(`document-file-(\d+)`) - - // Collect document data - type DocumentData struct { - File multipart.File - Header *multipart.FileHeader - Name string - Type string - Index int - } + // Second pass: process document metadata + for i, doc := range docsWithContent { + suffix := strings.TrimPrefix(doc.FormField, "document-file-") + nameField := "document-name-" + suffix + typeField := "document-type-" + suffix - var documents []DocumentData - - // First, identify all available indices - var indices []int - for key := range r.MultipartForm.File { - if matches := filePattern.FindStringSubmatch(key); len(matches) > 1 { - if index, err := strconv.Atoi(matches[1]); err == nil { - indices = append(indices, index) - } - } - } - - // Process each document - for _, index := range indices { - fileKey := fmt.Sprintf("document-file-%d", index) - nameKey := fmt.Sprintf("document-name-%d", index) - typeKey := fmt.Sprintf("document-type-%d", index) - - fileHeaders := r.MultipartForm.File[fileKey] - if len(fileHeaders) == 0 { - continue // Skip if no file uploaded - } - - fileHeader := fileHeaders[0] - file, err := fileHeader.Open() - if err != nil { - log.Printf("Error opening file %s: %v", fileHeader.Filename, err) - continue - } - - // Get document name (use filename if not provided) - documentName := r.FormValue(nameKey) - if documentName == "" { - documentName = fileHeader.Filename - } else { + // Get custom document name if provided + customName := formValues[nameField] + if customName != "" { // If a custom name is provided without extension, add the original file extension - if !strings.Contains(documentName, ".") { - extension := filepath.Ext(fileHeader.Filename) + if !strings.Contains(customName, ".") { + extension := filepath.Ext(doc.Name) if extension != "" { - documentName = documentName + extension + customName = customName + extension } } + docsWithContent[i].Name = customName } - log.Printf("Using document name: %s (original filename: %s)", documentName, fileHeader.Filename) - // Get document type - documentType := r.FormValue(typeKey) - if documentType == "" { - log.Printf("No document type for file %s", fileHeader.Filename) + docType := formValues[typeField] + if docType == "" { + log.Printf("No document type for file %s, skipping", doc.Name) continue } - // Log the document type for debugging - log.Printf("Document type for %s: '%s'", documentName, documentType) - - documents = append(documents, DocumentData{ - File: file, - Header: fileHeader, - Name: documentName, - Type: documentType, - Index: index, - }) + docsWithContent[i].Type = docType + log.Printf("Processing document: %s (type: %s) from field: %s", + docsWithContent[i].Name, docType, doc.FormField) } - if len(documents) == 0 { - http.Error(w, "No valid documents selected for upload", http.StatusBadRequest) - return + // Filter out documents with no type + var validDocs []DocumentWithContent + for _, doc := range docsWithContent { + if doc.Type != "" { + validDocs = append(validDocs, doc) + } } + docsWithContent = validDocs - // Read all file contents first to avoid keeping files open during concurrent uploads - type DocumentWithContent struct { - Name string - Type string - FileContent []byte - } + log.Printf("Total valid documents to upload: %d", len(docsWithContent)) - var docsWithContent []DocumentWithContent - for _, doc := range documents { - // Read file content - fileContent, err := io.ReadAll(doc.File) - if err != nil { - log.Printf("Error reading file %s: %v", doc.Header.Filename, err) - continue - } - doc.File.Close() // Close the file as soon as we're done with it - - docsWithContent = append(docsWithContent, DocumentWithContent{ - Name: doc.Name, - Type: doc.Type, - FileContent: fileContent, - }) + if len(docsWithContent) == 0 { + http.Error(w, "No valid documents selected for upload", http.StatusBadRequest) + return } // Concurrent upload with throttling