You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
514 lines
17 KiB
514 lines
17 KiB
package web
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/csv"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"math"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
root "marmic/servicetrade-toolbox"
|
|
"marmic/servicetrade-toolbox/internal/api"
|
|
"marmic/servicetrade-toolbox/internal/middleware"
|
|
)
|
|
|
|
// DocumentsHandler handles the document upload page
|
|
func DocumentsHandler(w http.ResponseWriter, r *http.Request) {
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
tmpl := root.WebTemplates
|
|
data := map[string]interface{}{
|
|
"Title": "Document Uploads",
|
|
"Session": session,
|
|
}
|
|
|
|
if r.Header.Get("HX-Request") == "true" {
|
|
// For HTMX requests, just send the document_upload partial
|
|
if err := tmpl.ExecuteTemplate(w, "document_upload", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
} else {
|
|
// For full page requests, first render document_upload into a buffer
|
|
var contentBuf bytes.Buffer
|
|
if err := tmpl.ExecuteTemplate(&contentBuf, "document_upload", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Add the rendered content to the data for the layout
|
|
data["BodyContent"] = contentBuf.String()
|
|
|
|
// Now render the layout with our content
|
|
if err := tmpl.ExecuteTemplate(w, "layout.html", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// ProcessCSVHandler processes a CSV file with job numbers
|
|
func ProcessCSVHandler(w http.ResponseWriter, r *http.Request) {
|
|
_, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
// Check if the request method is POST
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
// Parse the multipart form data with a 10MB limit
|
|
if err := r.ParseMultipartForm(10 << 20); err != nil {
|
|
http.Error(w, "Unable to parse form: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Get the file from the form
|
|
file, _, err := r.FormFile("csvFile")
|
|
if err != nil {
|
|
http.Error(w, "Error retrieving file: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
defer file.Close()
|
|
|
|
// Read the CSV data
|
|
csvData, err := csv.NewReader(file).ReadAll()
|
|
if err != nil {
|
|
http.Error(w, "Error reading CSV file: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if len(csvData) < 2 {
|
|
http.Error(w, "CSV file must contain at least a header row and one data row", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Find the index of the 'id' column
|
|
headerRow := csvData[0]
|
|
idColumnIndex := -1
|
|
for i, header := range headerRow {
|
|
if strings.ToLower(strings.TrimSpace(header)) == "id" {
|
|
idColumnIndex = i
|
|
break
|
|
}
|
|
}
|
|
|
|
// If 'id' column not found, try the first column
|
|
if idColumnIndex == -1 {
|
|
idColumnIndex = 0
|
|
log.Printf("No 'id' column found in CSV, using first column (header: %s)", headerRow[0])
|
|
} else {
|
|
log.Printf("Found 'id' column at index %d", idColumnIndex)
|
|
}
|
|
|
|
// Extract job numbers from the CSV
|
|
var jobNumbers []string
|
|
for rowIndex, row := range csvData {
|
|
// Skip header row
|
|
if rowIndex == 0 {
|
|
continue
|
|
}
|
|
|
|
if len(row) > idColumnIndex {
|
|
// Extract and clean up the job ID
|
|
jobID := strings.TrimSpace(row[idColumnIndex])
|
|
if jobID != "" {
|
|
jobNumbers = append(jobNumbers, jobID)
|
|
}
|
|
}
|
|
}
|
|
|
|
totalJobs := len(jobNumbers)
|
|
log.Printf("Extracted %d job IDs from CSV", totalJobs)
|
|
|
|
if totalJobs == 0 {
|
|
http.Error(w, "No valid job IDs found in the CSV file", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Create a hidden input with the job IDs
|
|
jobsValue := strings.Join(jobNumbers, ",")
|
|
|
|
// Generate HTML for the main response (hidden input for job-ids-container)
|
|
var responseHTML bytes.Buffer
|
|
responseHTML.WriteString(fmt.Sprintf(`<input type="hidden" name="jobNumbers" value="%s">`, jobsValue))
|
|
responseHTML.WriteString(fmt.Sprintf(`<p>Found <strong>%d</strong> job(s) in the CSV file</p>`, totalJobs))
|
|
|
|
// Generate out-of-band swap for the preview section - simplified version
|
|
responseHTML.WriteString(fmt.Sprintf(`
|
|
<div id="csv-preview" class="fade-me-out csv-preview-active" hx-swap-oob="true">
|
|
<h4>✓ Jobs Detected</h4>
|
|
<p>Upload to <strong>%d</strong> job(s)</p>
|
|
</div>
|
|
`, totalJobs))
|
|
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write(responseHTML.Bytes())
|
|
}
|
|
|
|
// UploadDocumentsHandler handles document uploads to jobs
|
|
func UploadDocumentsHandler(w http.ResponseWriter, r *http.Request) {
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
log.Printf("Starting document upload handler with Content-Length: %.2f MB",
|
|
float64(r.ContentLength)/(1024*1024))
|
|
|
|
maxMemory := int64(32 << 20) // 32MB in memory, rest to disk
|
|
if err := r.ParseMultipartForm(maxMemory); err != nil {
|
|
log.Printf("Error parsing multipart form: %v", err)
|
|
http.Error(w, "Unable to parse form: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
defer r.MultipartForm.RemoveAll() // Clean up temporary files
|
|
|
|
jobNumbers := r.FormValue("jobNumbers")
|
|
if jobNumbers == "" {
|
|
log.Printf("No job numbers found in hidden 'jobNumbers' input.")
|
|
http.Error(w, "No job numbers provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
log.Printf("Job numbers: %s", jobNumbers)
|
|
jobs := strings.Split(jobNumbers, ",")
|
|
if len(jobs) == 0 {
|
|
http.Error(w, "No valid job numbers provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Simple multi-upload: use original filenames as display names and default type "1"
|
|
fileHeaders := r.MultipartForm.File["documentFiles"]
|
|
if len(fileHeaders) == 0 {
|
|
http.Error(w, "No documents selected for upload", http.StatusBadRequest)
|
|
return
|
|
}
|
|
type FileToUploadMetadata struct {
|
|
OriginalFilename string
|
|
DisplayName string
|
|
Type string
|
|
TempFile string
|
|
File *os.File
|
|
}
|
|
var filesToUpload []FileToUploadMetadata
|
|
for _, fileHeader := range fileHeaders {
|
|
uploadedFile, err := fileHeader.Open()
|
|
if err != nil {
|
|
log.Printf("Error opening uploaded file %s: %v. Skipping.", fileHeader.Filename, err)
|
|
continue
|
|
}
|
|
metadata := FileToUploadMetadata{
|
|
OriginalFilename: fileHeader.Filename,
|
|
DisplayName: fileHeader.Filename,
|
|
Type: "1",
|
|
}
|
|
tempFileHandle, err := os.CreateTemp("", "upload-*"+filepath.Ext(fileHeader.Filename))
|
|
if err != nil {
|
|
log.Printf("Error creating temp file for %s: %v. Skipping.", fileHeader.Filename, err)
|
|
uploadedFile.Close()
|
|
continue
|
|
}
|
|
if _, err := io.Copy(tempFileHandle, uploadedFile); err != nil {
|
|
log.Printf("Error copying to temp file for %s: %v. Skipping.", fileHeader.Filename, err)
|
|
uploadedFile.Close()
|
|
tempFileHandle.Close()
|
|
os.Remove(tempFileHandle.Name())
|
|
continue
|
|
}
|
|
uploadedFile.Close()
|
|
if _, err := tempFileHandle.Seek(0, 0); err != nil {
|
|
log.Printf("Error seeking temp file for %s: %v. Skipping.", fileHeader.Filename, err)
|
|
tempFileHandle.Close()
|
|
os.Remove(tempFileHandle.Name())
|
|
continue
|
|
}
|
|
metadata.TempFile = tempFileHandle.Name()
|
|
metadata.File = tempFileHandle
|
|
filesToUpload = append(filesToUpload, metadata)
|
|
}
|
|
activeFilesProcessedCount := len(filesToUpload)
|
|
if activeFilesProcessedCount == 0 {
|
|
log.Println("No files processed for upload.")
|
|
http.Error(w, "No documents were processed for upload.", http.StatusBadRequest)
|
|
return
|
|
}
|
|
log.Printf("Total active files prepared for upload: %d", activeFilesProcessedCount)
|
|
|
|
const maxConcurrent = 5
|
|
const requestDelay = 300 * time.Millisecond
|
|
|
|
type UploadResult struct {
|
|
JobID string
|
|
DocName string
|
|
Success bool
|
|
Error string
|
|
Data map[string]interface{}
|
|
FileSize int64
|
|
}
|
|
|
|
totalUploads := len(jobs) * activeFilesProcessedCount
|
|
resultsChan := make(chan UploadResult, totalUploads)
|
|
var wg sync.WaitGroup
|
|
semaphore := make(chan struct{}, maxConcurrent)
|
|
|
|
log.Printf("Starting up to %d concurrent upload workers for %d total uploads (%d jobs x %d active files)",
|
|
maxConcurrent, totalUploads, len(jobs), activeFilesProcessedCount)
|
|
|
|
for _, jobID := range jobs {
|
|
for _, metadataToUpload := range filesToUpload {
|
|
currentUploadMetadata := metadataToUpload
|
|
wg.Add(1)
|
|
go func(jobID string, meta FileToUploadMetadata) {
|
|
defer wg.Done()
|
|
semaphore <- struct{}{}
|
|
defer func() { <-semaphore }()
|
|
time.Sleep(requestDelay)
|
|
|
|
fileNameForUpload := meta.DisplayName
|
|
fileHandleForUpload, err := os.Open(meta.TempFile)
|
|
if err != nil {
|
|
log.Printf("Goroutine Error: Failed to re-open temp file %s for job %s (uploading as %s): %v",
|
|
meta.TempFile, jobID, fileNameForUpload, err)
|
|
resultsChan <- UploadResult{
|
|
JobID: jobID,
|
|
DocName: fileNameForUpload,
|
|
Success: false,
|
|
Error: fmt.Sprintf("Error preparing file for upload: %v", err),
|
|
FileSize: 0,
|
|
}
|
|
return
|
|
}
|
|
defer fileHandleForUpload.Close()
|
|
|
|
fileInfo, statErr := fileHandleForUpload.Stat()
|
|
var expectedSize int64
|
|
if statErr == nil {
|
|
expectedSize = fileInfo.Size()
|
|
} else {
|
|
log.Printf("Goroutine Warning: Failed to get file info for %s (original: %s, job %s, uploading as %s): %v",
|
|
meta.TempFile, meta.OriginalFilename, jobID, fileNameForUpload, statErr)
|
|
}
|
|
|
|
if len(jobs) > 10 {
|
|
jitter := time.Duration(100+(time.Now().UnixNano()%400)) * time.Millisecond
|
|
time.Sleep(jitter)
|
|
}
|
|
|
|
sizeTracker := &readCloserWithSize{reader: fileHandleForUpload, size: 0}
|
|
log.Printf("Goroutine Info: Starting to stream file %s (original: %s, uploading as %s, type: %s) to job %s from temp file %s",
|
|
meta.TempFile, meta.OriginalFilename, fileNameForUpload, meta.Type, jobID, meta.TempFile)
|
|
|
|
// Define uploadStart here for per-goroutine timing
|
|
uploadStartGoroutine := time.Now()
|
|
uploadResultData, errUpload := session.UploadAttachmentFile(jobID, fileNameForUpload, meta.Type, sizeTracker)
|
|
uploadDuration := time.Since(uploadStartGoroutine)
|
|
|
|
fileSizeUploaded := sizeTracker.Size()
|
|
sizeMatch := true
|
|
if expectedSize > 0 && math.Abs(float64(expectedSize-fileSizeUploaded)) > float64(expectedSize)*0.05 {
|
|
sizeMatch = false
|
|
log.Printf("Goroutine WARNING: Size mismatch for %s (original: %s, uploaded as %s) to job %s. Expected: %d, Uploaded: %d",
|
|
meta.TempFile, meta.OriginalFilename, fileNameForUpload, jobID, expectedSize, fileSizeUploaded)
|
|
}
|
|
|
|
if errUpload != nil {
|
|
log.Printf("Goroutine Error: Uploading %s (original: %s, as %s) to job %s failed after %v: %v",
|
|
meta.TempFile, meta.OriginalFilename, fileNameForUpload, jobID, uploadDuration, errUpload)
|
|
resultsChan <- UploadResult{
|
|
JobID: jobID,
|
|
DocName: fileNameForUpload,
|
|
Success: false,
|
|
Error: errUpload.Error(),
|
|
FileSize: fileSizeUploaded,
|
|
}
|
|
} else if !sizeMatch {
|
|
log.Printf("Goroutine Error: Upload of %s (original: %s, as %s) to job %s appears corrupted. API reported success but file sizes mismatch.",
|
|
meta.TempFile, meta.OriginalFilename, fileNameForUpload, jobID)
|
|
resultsChan <- UploadResult{
|
|
JobID: jobID,
|
|
DocName: fileNameForUpload,
|
|
Success: false,
|
|
Error: "Upload appears corrupted (file size mismatch)",
|
|
FileSize: fileSizeUploaded,
|
|
}
|
|
} else {
|
|
log.Printf("Goroutine Success: Uploaded %s (original: %s, %.2f MB, as %s, type: %s) to job %s in %v",
|
|
meta.TempFile, meta.OriginalFilename, float64(fileSizeUploaded)/(1024*1024), fileNameForUpload, meta.Type, jobID, uploadDuration)
|
|
resultsChan <- UploadResult{
|
|
JobID: jobID,
|
|
DocName: fileNameForUpload,
|
|
Success: true,
|
|
Data: uploadResultData,
|
|
FileSize: fileSizeUploaded,
|
|
}
|
|
}
|
|
}(jobID, currentUploadMetadata)
|
|
}
|
|
}
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
close(resultsChan)
|
|
log.Println("All upload goroutines finished.")
|
|
}()
|
|
|
|
results := make(map[string][]UploadResult)
|
|
resultsCount := 0
|
|
var totalBytesUploaded int64
|
|
|
|
for result := range resultsChan {
|
|
resultsCount++
|
|
log.Printf("Received result %d/%d: Job %s, File %s, Success: %v, Size: %.2f MB",
|
|
resultsCount, totalUploads, result.JobID, result.DocName, result.Success,
|
|
float64(result.FileSize)/(1024*1024))
|
|
if result.Success {
|
|
totalBytesUploaded += result.FileSize
|
|
}
|
|
if _, exists := results[result.JobID]; !exists {
|
|
results[result.JobID] = []UploadResult{}
|
|
}
|
|
results[result.JobID] = append(results[result.JobID], result)
|
|
}
|
|
|
|
log.Printf("All results collected. Total: %d, Total bytes uploaded: %.2f MB",
|
|
resultsCount, float64(totalBytesUploaded)/(1024*1024))
|
|
|
|
var resultHTML bytes.Buffer
|
|
var totalSuccess, totalFailure int
|
|
for _, jobResults := range results {
|
|
for _, result := range jobResults {
|
|
if result.Success {
|
|
totalSuccess++
|
|
} else {
|
|
totalFailure++
|
|
}
|
|
}
|
|
}
|
|
|
|
resultHTML.WriteString("<div class=\"upload-summary\">")
|
|
resultHTML.WriteString("<h3>Upload Results</h3>")
|
|
resultHTML.WriteString("<div class=\"upload-stats\">")
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-box\"><div class=\"stat-value\">%d</div><div class=\"stat-label\">Total Jobs</div></div>", len(results)))
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-box success-stat\"><div class=\"stat-value\">%d</div><div class=\"stat-label\">Successful Uploads</div></div>", totalSuccess))
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-box error-stat\"><div class=\"stat-value\">%d</div><div class=\"stat-label\">Failed Uploads</div></div>", totalFailure))
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-box\"><div class=\"stat-value\">%d</div><div class=\"stat-label\">Files Processed</div></div>", resultsCount))
|
|
resultHTML.WriteString("</div>")
|
|
|
|
if totalFailure == 0 && resultsCount > 0 {
|
|
resultHTML.WriteString("<p>All documents were successfully uploaded to ServiceTrade!</p>")
|
|
} else if resultsCount == 0 {
|
|
resultHTML.WriteString("<p>No documents were processed for upload.</p>")
|
|
} else {
|
|
resultHTML.WriteString("<p>Some documents failed to upload. See details below.</p>")
|
|
}
|
|
resultHTML.WriteString("</div>")
|
|
|
|
resultHTML.WriteString("<div class=\"job-results\">")
|
|
sortedJobs := make([]string, 0, len(results))
|
|
for jobID := range results {
|
|
sortedJobs = append(sortedJobs, jobID)
|
|
}
|
|
sort.Strings(sortedJobs)
|
|
|
|
for _, jobID := range sortedJobs {
|
|
jobResults := results[jobID]
|
|
jobHasSuccess := false
|
|
jobHasFailure := false
|
|
for _, result := range jobResults {
|
|
if result.Success {
|
|
jobHasSuccess = true
|
|
} else {
|
|
jobHasFailure = true
|
|
}
|
|
}
|
|
jobClass := "neutral"
|
|
if jobHasSuccess && !jobHasFailure {
|
|
jobClass = "success"
|
|
} else if jobHasFailure {
|
|
jobClass = "error"
|
|
}
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"job-result %s\">", jobClass))
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"job-id\">Job ID: %s</div>", jobID))
|
|
if len(jobResults) > 0 {
|
|
resultHTML.WriteString("<div class=\"file-results\">")
|
|
sort.Slice(jobResults, func(i, j int) bool {
|
|
return jobResults[i].DocName < jobResults[j].DocName
|
|
})
|
|
for _, result := range jobResults {
|
|
fileClass := "success"
|
|
icon := "✓"
|
|
message := "Successfully uploaded"
|
|
if !result.Success {
|
|
fileClass = "error"
|
|
icon = "✗"
|
|
message = strings.ReplaceAll(result.Error, "<", "<")
|
|
message = strings.ReplaceAll(message, ">", ">")
|
|
}
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"file-result %s\">", fileClass))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"status-icon\">%s</span>", icon))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"file-name\">%s:</span>", result.DocName))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"file-message\">%s</span>", message))
|
|
resultHTML.WriteString("</div>")
|
|
}
|
|
resultHTML.WriteString("</div>")
|
|
} else {
|
|
resultHTML.WriteString("<p>No file upload results for this job.</p>")
|
|
}
|
|
resultHTML.WriteString("</div>")
|
|
}
|
|
resultHTML.WriteString("</div>")
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write(resultHTML.Bytes())
|
|
}
|
|
|
|
// readCloserWithSize is a custom io.Reader that counts the bytes read
|
|
type readCloserWithSize struct {
|
|
reader io.ReadCloser
|
|
size int64
|
|
}
|
|
|
|
func (r *readCloserWithSize) Read(p []byte) (n int, err error) {
|
|
n, err = r.reader.Read(p)
|
|
r.size += int64(n)
|
|
return n, err
|
|
}
|
|
|
|
func (r *readCloserWithSize) Close() error {
|
|
if r.reader != nil {
|
|
return r.reader.Close()
|
|
}
|
|
return nil // Allow closing nil reader safely
|
|
}
|
|
|
|
// Size returns the current size of data read
|
|
func (r *readCloserWithSize) Size() int64 {
|
|
return r.size
|
|
}
|
|
|
|
// DocumentFieldAddHandler and DocumentFieldRemoveHandler are REMOVED
|
|
// as they are no longer needed with the multi-file input and new chip UI.
|
|
|