diff --git a/api/controllers/cac.js b/api/controllers/cac.js index 6cf65590..db11c71d 100644 --- a/api/controllers/cac.js +++ b/api/controllers/cac.js @@ -8,19 +8,33 @@ exports.protectedOptions = function (args, res) { res.status(200).send(); }; -exports.publicCACSignUp = async function (args, res) { - const CACUser = mongoose.model('CACUser'); - const Project = mongoose.model('Project'); +const MAX_EMAIL_LENGTH = 254; +const EMAIL_SHAPE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +// These routes are anonymous, so anything but a plain address string (an operator object such as +// {"$ne": ""}) must never reach a query. Returns the trimmed address, or null. +function cleanEmail(value) { + if (typeof value !== 'string') { + return null; + } + const email = value.trim(); + return email.length <= MAX_EMAIL_LENGTH && EMAIL_SHAPE.test(email) ? email : null; +} - let cacObject = args.swagger.params.cac.value; +exports.publicCACSignUp = async function (args, res) { + const cacObject = args.swagger.params.cac.value; const projectId = args.swagger.params.projId.value; defaultLog.info('publicCACSignUp for project:', projectId); - if (!mongoose.Types.ObjectId.isValid(projectId)) { + const email = cleanEmail(cacObject && cacObject.email); + if (!Utils.isObjectIdString(projectId) || !email) { return Actions.sendResponse(res, 400, {}); } + const CACUser = mongoose.model('CACUser'); + const Project = mongoose.model('Project'); delete cacObject.read; delete cacObject.write; + cacObject.email = email; cacObject.project = projectId; try { @@ -60,20 +74,20 @@ exports.publicCACSignUp = async function (args, res) { }; exports.publicCACRemoveMember = async function (args, res) { - const Project = mongoose.model('Project'); - const CACUser = mongoose.model('CACUser'); - const projId = args.swagger.params.projId.value; defaultLog.info('publicCACRemoveMember for project:', projId); - if (!mongoose.Types.ObjectId.isValid(projId)) { + const cac = args.swagger.params.cac.value; + const email = cleanEmail(cac && cac.email); + if (!Utils.isObjectIdString(projId) || !email) { return Actions.sendResponse(res, 400, {}); } - const cac = args.swagger.params.cac.value; + const Project = mongoose.model('Project'); + const CACUser = mongoose.model('CACUser'); try { const member = await CACUser.findOne({ _schemaName: 'CACUser', - email: cac.email, + email: email, project: new mongoose.Types.ObjectId(projId) }); @@ -85,10 +99,10 @@ exports.publicCACRemoveMember = async function (args, res) { await CACUser.deleteOne({ _id: member._id }); Utils.recordAction('Delete', 'CACMemberFromProject', 'public', member._id); - defaultLog.info('Removed CAC member email:', cac.email, 'from project:', projId); + defaultLog.info('Removed CAC member %s from project %s', member._id, projId); return Actions.sendResponse(res, 200, {}); } else { - defaultLog.info('CAC member not found with email:', cac.email, 'in project:', projId); + defaultLog.info('CAC member to remove not found in project %s', projId); return Actions.sendResponse(res, 404, {}); } } catch (e) { diff --git a/api/controllers/comment.js b/api/controllers/comment.js index 86dd31f5..79f054fa 100644 --- a/api/controllers/comment.js +++ b/api/controllers/comment.js @@ -4,7 +4,6 @@ var Actions = require('../helpers/actions'); var Utils = require('../helpers/utils'); const demiPush = require('../helpers/demiPush'); const parentRead = require('../helpers/parentRead'); -const { DateTime } = require('luxon'); const { COMMENT_FIELDS: ALLOWED_FIELDS, COMMENT_PUBLIC_FIELDS: PUBLIC_FIELDS, PUBLIC_ROLES } = require('../helpers/constants'); // A comment's own read[] stays Published when staff unpublish its period or project, so the @@ -382,70 +381,6 @@ async function getNextCommentIdCount(period) { return periodUpdated.commentIdCount; } -// Create a new Comment -exports.unProtectedPost = async function (args, res) { - try { - const obj = args.swagger.params.comment.value; - defaultLog.info('Incoming new comment for period: %s', obj.period); - if (args.swagger.params.period && args.swagger.params.period.value && !mongoose.Types.ObjectId.isValid(args.swagger.params.period.value)) { - return Actions.sendResponse(res, 400, { }); - } - const Comment = mongoose.model('Comment'); - - // Ensure the comment is received before the end date of the period. - const commentPeriodModel = mongoose.model('CommentPeriod'); - const period = await commentPeriodModel.findOne({ _id: new mongoose.Types.ObjectId(obj.period) }); - - defaultLog.info('Period: %s', period && period._id); - - if (DateTime.now() > DateTime.fromJSDate(new Date(period.dateCompleted))) { - // Past the specified time - const endDate = DateTime.fromJSDate(new Date(period.dateCompleted)).toFormat('MMM dd, yyyy h:mm a'); - defaultLog.info('Comment was not received before completed date:', endDate); - return Actions.sendResponse(res, - 400, { - code: '400', - message: 'Comment recieved after date completed' - } - ); - } - - // get the next commentID for this period - const commentIdCount = await getNextCommentIdCount(new mongoose.Types.ObjectId(obj.period)); - - defaultLog.info('Next comment id: %d', commentIdCount); - - let cmt = new Comment(); - cmt._schemaName = 'Comment'; - cmt.eaoStatus = 'Pending'; - cmt.author = obj.author; - cmt.comment = obj.comment; - cmt.dateAdded = new Date(); - cmt.dateUpdated = new Date(); - cmt.isAnonymous = obj.isAnonymous; - cmt.location = obj.location; - cmt.period = new mongoose.Types.ObjectId(obj.period); - cmt.commentId = commentIdCount; - cmt.documents = []; - cmt.datePosted = undefined; - cmt.submittedCAC = obj.submittedCAC; - - cmt.read = ['staff', 'sysadmin']; - cmt.write = ['staff', 'sysadmin']; - cmt.delete = ['staff', 'sysadmin']; - - defaultLog.debug('About to save comment for period: %s', obj.period); - - const c = await cmt.save(); - Utils.recordAction('Post', 'Comment', 'public', c._id); - defaultLog.info('Saved new comment object: %s', c._id); - return Actions.sendMirrored(res, 200, c, demiPush.comment(c)); - } catch (e) { - defaultLog.error(`Error: ${e.message}`); - return Actions.sendResponse(res, 400, e); - } -}; - // Update an existing Comment exports.protectedPut = async function (args, res) { var objId = args.swagger.params.commentId.value; diff --git a/api/controllers/document.js b/api/controllers/document.js index 52b5483e..e4af3b6d 100644 --- a/api/controllers/document.js +++ b/api/controllers/document.js @@ -110,107 +110,6 @@ exports.publicGet = async function (args, res,) { } }; -exports.unProtectedPost = async function (args, res) { - defaultLog.info('Creating new public document object'); - if (args.swagger.params._comment && args.swagger.params._comment.value && !mongoose.Types.ObjectId.isValid(args.swagger.params._comment.value)) { - return Actions.sendResponse(res, 400, { }); - } - if (args.swagger.params.project && args.swagger.params.project.value && !mongoose.Types.ObjectId.isValid(args.swagger.params.project.value)) { - return Actions.sendResponse(res, 400, { }); - } - - var _comment = args.swagger.params._comment.value; - var project = args.swagger.params.project.value; - var upfile = args.swagger.params.upfile.value; - - // Enforce file size limit of 10MB in RAM - const MAX_FILE_SIZE = 10 * 1024 * 1024; - if (upfile.size > MAX_FILE_SIZE) { - defaultLog.warn('File upload rejected: exceeds 10MB limit (%d bytes)', upfile.size); - return Actions.sendResponse(res, 400, { message: 'File size exceeds 10MB limit.' }); - } - - // Enforce MIME allow-list - const allowedMimeTypes = [ - 'application/pdf', - 'image/png', - 'image/jpeg', - 'image/jpg', - 'application/msword', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - ]; - if (!allowedMimeTypes.includes(upfile.mimetype)) { - defaultLog.warn('File upload rejected: unallowed MIME type (%s)', upfile.mimetype); - return Actions.sendResponse(res, 400, { message: 'MIME type is not allowed.' }); - } - - var guid = crypto.randomUUID(); - var ext = mime.extension(upfile.mimetype); - var tempFilePath = uploadDir + guid + '.' + ext; - - try { - // Virus scan - if (ENABLE_VIRUS_SCANNING) { - var scanPassed = await Utils.avScan(upfile.buffer); - if (!scanPassed) { - defaultLog.warn('File failed virus check.'); - return Actions.sendResponse(res, 400, { 'message': 'File failed virus check.' }); - } - } - - // Write to temp file and upload to MinIO - fs.writeFileSync(tempFilePath, upfile.buffer); - var minioFile = await MinioController.putDocument( - MinioController.BUCKETS.DOCUMENTS_BUCKET, project, upfile.originalname, tempFilePath - ); - fs.unlinkSync(tempFilePath); - - // Save document to MongoDB - var Document = mongoose.model('Document'); - var doc = new Document(); - doc.project = new mongoose.Types.ObjectId(project); - doc._comment = new mongoose.Types.ObjectId(_comment); - doc._addedBy = 'public'; - doc._createdDate = new Date(); - doc.read = ['sysadmin', 'staff']; - doc.write = ['sysadmin', 'staff']; - doc.delete = ['sysadmin', 'staff']; - - doc.internalOriginalName = upfile.originalname; - doc.internalURL = minioFile.path; - doc.internalExt = minioFile.extension; - doc.internalSize = upfile.size; - doc.internalOriginalSha256 = sha256Of(upfile.buffer); - doc.passedAVCheck = true; - doc.internalMime = upfile.mimetype; - - doc.documentSource = 'COMMENT'; - doc.displayName = upfile.originalname; - doc.documentFileName = upfile.originalname; - doc.dateUploaded = new Date(); - doc.datePosted = new Date(); - doc.documentAuthor = args.body.documentAuthor; - doc.documentAuthorType = new mongoose.Types.ObjectId(args.body.documentAuthorType); - - try { - var d = await doc.save(); - defaultLog.info('Saved new document object:', d._id); - var Comment = mongoose.model('Comment'); - await Comment.updateOne({ _id: _comment }, { $addToSet: { documents: d._id } }); - Utils.recordAction('Post', 'Document', 'public', d._id); - return Actions.sendMirrored(res, 200, d, demiPush.document(d)); - } catch (saveError) { - defaultLog.error('Document save failed, rolling back MinIO:', saveError); - MinioController.deleteDocument(MinioController.BUCKETS.DOCUMENTS_BUCKET, doc.project, doc.internalURL); - return Actions.sendResponse(res, 400, saveError); - } - } catch (e) { - defaultLog.error(`Error in unProtectedPost: ${e.message}`); - delete e['path']; - return Actions.sendResponse(res, 400, e); - } -}; - exports.protectedHead = function (args, res) { defaultLog.debug('Roles:', args.swagger.params.auth_payload.realm_access.roles); @@ -575,15 +474,23 @@ exports.protectedOpen = function (args, res) { }; // Create a new document +// The admin upload form sends the string "null" for these when they are empty. +const NULLABLE_ID_FIELDS = ['milestone', 'type', 'documentAuthorType', 'projectPhase']; + exports.protectedPost = async function (args, res) { defaultLog.info('Creating new protected document object'); var project = args.swagger.params.project.value; var _comment = args.swagger.params._comment.value; - if (args.swagger.params._comment && args.swagger.params._comment.value && !mongoose.Types.ObjectId.isValid(_comment)) { - return Actions.sendResponse(res, 400, { }); - } - if (args.swagger.params.project && args.swagger.params.project.value && !mongoose.Types.ObjectId.isValid(project)) { + const badId = ['project', '_comment', ...NULLABLE_ID_FIELDS].find(name => { + const value = args.swagger.params[name] && args.swagger.params[name].value; + if (!value || (value === 'null' && NULLABLE_ID_FIELDS.includes(name))) { + return false; + } + return !Utils.isObjectIdString(value); + }); + if (badId) { + defaultLog.warn('Protected file upload rejected: %s is not an id', badId); return Actions.sendResponse(res, 400, { }); } @@ -622,10 +529,14 @@ exports.protectedPost = async function (args, res) { // Write to temp file and upload to MinIO fs.writeFileSync(tempFilePath, upfile.buffer); - var minioFile = await MinioController.putDocument( - MinioController.BUCKETS.DOCUMENTS_BUCKET, project, args.swagger.params.documentFileName.value, tempFilePath - ); - fs.unlinkSync(tempFilePath); + let minioFile; + try { + minioFile = await MinioController.putDocument( + MinioController.BUCKETS.DOCUMENTS_BUCKET, project, args.swagger.params.documentFileName.value, tempFilePath + ); + } finally { + fs.unlinkSync(tempFilePath); + } // Build document model var Document = mongoose.model('Document'); @@ -689,7 +600,17 @@ exports.protectedPost = async function (args, res) { return Actions.sendMirrored(res, 200, d, demiPush.document(d)); } catch (saveError) { defaultLog.error('Document save failed, rolling back MinIO:', saveError); - MinioController.deleteDocument(MinioController.BUCKETS.DOCUMENTS_BUCKET, doc.project, doc.internalURL); + try { + // A save can report failure after the write landed (a write timeout); then the file is in use. + if (await Document.exists({ _id: doc._id })) { + defaultLog.warn('Document %s was stored despite the save error; keeping its file', doc._id); + } else { + // internalURL already starts with the project, so it is the whole object key. + await MinioController.deleteObject(MinioController.BUCKETS.DOCUMENTS_BUCKET, doc.internalURL); + } + } catch (rollbackError) { + defaultLog.error('MinIO rollback of %s failed: %s', doc.internalURL, rollbackError); + } return Actions.sendResponse(res, 400, saveError); } } catch (e) { diff --git a/api/helpers/minio.js b/api/helpers/minio.js index 251bc839..77431641 100644 --- a/api/helpers/minio.js +++ b/api/helpers/minio.js @@ -154,17 +154,19 @@ exports.putFileDirect = putFileDirect; * @param fileName the name of the file */ var deleteDocument = function (bucket, projectCode, fileName) { - if(isMockService()) return Promise.resolve(() => { return {}; }); - return minioClient.removeObject(bucket, projectCode + '/' + fileName) - .then(function (result, err) { - if (err) { - throw err; - } - return result; - }); + return deleteObject(bucket, projectCode + '/' + fileName); }; exports.deleteDocument = deleteDocument; +/** + * Delete a file from Minio by its full object key, which is what a document's internalURL holds. + */ +const deleteObject = function (bucket, objectName) { + if(isMockService()) return Promise.resolve(); + return minioClient.removeObject(bucket, objectName); +}; +exports.deleteObject = deleteObject; + /** * Get a Minio presigned url, for a specific file object, that permits GET operations. * The url can be used multiple times, but expires after 5 minutes. diff --git a/api/helpers/utils.js b/api/helpers/utils.js index e553db3d..057dd9f9 100644 --- a/api/helpers/utils.js +++ b/api/helpers/utils.js @@ -93,6 +93,14 @@ exports.getValidObjectId = function (param) { return new mongoose.Types.ObjectId(val); }; +/** + * True only for a 24-hex-digit string. ObjectId.isValid also passes any 12-character string and + * some objects, which lets a request body steer a query. + */ +exports.isObjectIdString = function (value) { + return typeof value === 'string' && /^[0-9a-f]{24}$/i.test(value); +}; + exports.getBasePath = function (protocol, host) { return protocol + '://' + host; }; diff --git a/api/swagger/swagger.yaml b/api/swagger/swagger.yaml index 98aedf47..6d1b1bc2 100644 --- a/api/swagger/swagger.yaml +++ b/api/swagger/swagger.yaml @@ -4020,44 +4020,6 @@ paths: description: "Access Denied" schema: $ref: "#/definitions/Error" - post: - tags: - - document - summary: "Create a new document from public" - operationId: unProtectedPost - description: "Create a new document object from public." - consumes: - - multipart/form-data - parameters: - - name: upfile - in: formData - description: "The file to upload." - required: true - type: file - - name: project - in: formData - description: "The project to bind this document to." - required: false - type: string - - name: _comment - in: formData - description: "The comment to bind this document to." - required: false - type: string - - name: displayName - in: formData - description: "The displayName of the file." - required: false - type: string - responses: - "200": - description: "Success" - schema: - $ref: "#/definitions/DocObject" - "403": - description: "Access Denied" - schema: - $ref: "#/definitions/Error" get: tags: - document @@ -5295,27 +5257,6 @@ paths: description: "Access Denied" schema: $ref: "#/definitions/Error" - post: - tags: - - comment - summary: "Create a new comment" - operationId: unProtectedPost - description: "Create a new comment object." - parameters: - - name: comment - in: body - required: true - schema: - $ref: "#/definitions/CommentObject" - responses: - "200": - description: "Success" - schema: - $ref: "#/definitions/CommentObject" - "403": - description: "Access Denied" - schema: - $ref: "#/definitions/Error" head: tags: - comment diff --git a/test/controllers/analytics-callsites.test.js b/test/controllers/analytics-callsites.test.js index 4913ca2f..3edad9a5 100644 --- a/test/controllers/analytics-callsites.test.js +++ b/test/controllers/analytics-callsites.test.js @@ -172,13 +172,6 @@ describe('Analytics call sites', () => { }); }); - it('document.unProtectedPost audits nothing, so a public write cannot land as a staff row', async () => { - await documentController.unProtectedPost(docArgs(), res); - - expect(res.status.args, `expected 200, got ${JSON.stringify(res.status.args)}`).to.deep.equal([[200]]); - expect(analytics.auditEvent.called).to.be.false; - }); - it('document.protectedGet is a read path and audits nothing', async () => { sinon.stub(Utils, 'runDataQuery').resolves([{ _id: OID }]); const args = docArgs(); diff --git a/test/controllers/cac.test.js b/test/controllers/cac.test.js index 82eb9bd2..a6ed6806 100644 --- a/test/controllers/cac.test.js +++ b/test/controllers/cac.test.js @@ -186,5 +186,72 @@ describe('CAC Controller', () => { expect(res.status.calledWith(400)).to.be.true; expect(defaultLog.error.called).to.be.true; }); + + it('logs the member id, never the submitted address', async () => { + cacUserModel.findOne.resolves({ _id: VALID_MEMBER_ID, email: 'test@example.com' }); + projectModel.updateOne.resolves({ modifiedCount: 1 }); + await cac.publicCACRemoveMember(makeArgs(), res); + expect(JSON.stringify(defaultLog.info.args)).to.not.include('test@example.com'); + }); + + it('logs a miss without the submitted address', async () => { + cacUserModel.findOne.resolves(null); + await cac.publicCACRemoveMember(makeArgs(), res); + expect(JSON.stringify(defaultLog.info.args)).to.not.include('test@example.com'); + }); + }); + + // --------------------------------------------------------------------------- + // Both routes are anonymous: a body field shaped as a query operator must not reach CACUser. + describe('anonymous input that could steer the member query', () => { + const handlers = { + publicCACSignUp: cac.publicCACSignUp, + publicCACRemoveMember: cac.publicCACRemoveMember + }; + const badInput = { + 'an operator object as email': { cac: { value: { email: { $ne: '' } } } }, + 'a regex object as email': { cac: { value: { email: { $regex: '.*' } } } }, + 'an array as email': { cac: { value: { email: ['test@example.com'] } } }, + 'an email that is not an address': { cac: { value: { email: 'not-an-address' } } }, + 'an email over 254 characters': { cac: { value: { email: `${'a'.repeat(250)}@example.com` } } }, + 'an email of exactly 255 characters': { cac: { value: { email: `${'a'.repeat(243)}@example.com` } } }, + 'no email': { cac: { value: {} } }, + 'a 12-character project id': { projId: { value: 'abcdefghijkl' } } + }; + + Object.entries(handlers).forEach(([name, handler]) => { + Object.entries(badInput).forEach(([label, overrides]) => { + it(`${name} refuses ${label} with 400 before any query`, async () => { + await handler(makeArgs(overrides), res); + + expect(res.status.args).to.deep.equal([[400]]); + expect(cacUserModel.findOne.called, 'CACUser queried').to.be.false; + expect(cacUserModel.deleteOne.called, 'CACUser deleted').to.be.false; + }); + }); + + it(`${name} looks the member up by the trimmed address string`, async () => { + cacUserModel.findOne.resolves({ _id: VALID_MEMBER_ID, email: 'test@example.com' }); + projectModel.findOneAndUpdate.resolves(null); + projectModel.updateOne.resolves({ modifiedCount: 1 }); + + await handler(makeArgs({ cac: { value: { email: ' test@example.com ' } } }), res); + + expect(res.status.args).to.deep.equal([[200]]); + expect(cacUserModel.findOne.firstCall.args[0].email).to.equal('test@example.com'); + }); + + it(`${name} accepts an email of exactly 254 characters`, async () => { + const email = `${'a'.repeat(242)}@example.com`; + cacUserModel.findOne.resolves({ _id: VALID_MEMBER_ID, email }); + projectModel.findOneAndUpdate.resolves(null); + projectModel.updateOne.resolves({ modifiedCount: 1 }); + + await handler(makeArgs({ cac: { value: { email } } }), res); + + expect(res.status.args).to.deep.equal([[200]]); + expect(cacUserModel.findOne.firstCall.args[0].email).to.equal(email); + }); + }); }); }); diff --git a/test/controllers/comment.test.js b/test/controllers/comment.test.js index f306ba85..1d7b4458 100644 --- a/test/controllers/comment.test.js +++ b/test/controllers/comment.test.js @@ -10,7 +10,6 @@ const { expect } = require('chai'); const sinon = require('sinon'); const mongoose = require('mongoose'); const util = require('util'); -const { DateTime } = require('luxon'); const Utils = require('../../api/helpers/utils'); const parentRead = require('../../api/helpers/parentRead'); const Actions = require('../../api/helpers/actions'); @@ -362,19 +361,9 @@ describe('Comment Controller', () => { beforeEach(() => { sinon.stub(defaultLog, 'debug'); - cpModel.findOne.resolves({ _id: VALID_CP_ID, dateCompleted: DateTime.now().plus({ days: 2 }).toJSDate() }); cpModel.findOneAndUpdate.resolves({ commentIdCount: 15 }); }); - it('logs a public submission without its author, location or text', async () => { - useMockComment(); - - await commentController.unProtectedPost(withComment(), res); - - expect(res.status.calledWith(200)).to.be.true; - expectNoBody(); - }); - it('logs a staff-entered comment without its body', async () => { useMockComment(); @@ -437,40 +426,6 @@ describe('Comment Controller', () => { }); }); - describe('unProtectedPost', () => { - it('accepts comment before end date', async () => { - const args = makeArgs(); - // Future date completion - const futureDate = DateTime.now().plus({ days: 2 }).toJSDate(); - cpModel.findOne.resolves({ dateCompleted: futureDate }); - cpModel.findOneAndUpdate.resolves({ commentIdCount: 15 }); - - function MockComment(obj) { - Object.assign(this, obj); - this.save = sinon.stub().resolves({ _id: VALID_COMM_ID }); - } - mongoose.model.restore(); - sinon.stub(mongoose, 'model').callsFake(name => { - if (name === 'Comment') return MockComment; - if (name === 'CommentPeriod') return cpModel; - return {}; - }); - - await commentController.unProtectedPost(args, res); - expect(res.status.calledWith(200)).to.be.true; - }); - - it('rejects comment after end date', async () => { - const args = makeArgs(); - // Past date completion - const pastDate = DateTime.now().minus({ days: 2 }).toJSDate(); - cpModel.findOne.resolves({ dateCompleted: pastDate }); - - await commentController.unProtectedPost(args, res); - expect(res.status.calledWith(400)).to.be.true; - }); - }); - describe('protectedPut', () => { it('updates comment values', async () => { const args = makeArgs(); diff --git a/test/controllers/demi-push-await.test.js b/test/controllers/demi-push-await.test.js index 4a9748fb..ad7a5730 100644 --- a/test/controllers/demi-push-await.test.js +++ b/test/controllers/demi-push-await.test.js @@ -78,7 +78,7 @@ describe('DEMI push awaited before the reply', () => { ['document POST', 'document', 'documents', 'Document', () => documentController.protectedPost(docArgs(), res), () => {}, savedBody], ['document publish', 'document', 'documents', 'Document', () => documentController.protectedPublish(docArgs(), res), () => {}, savedBody], ['project PUT', 'project', 'projects', 'Project', () => projectController.protectedPut(projArgs(), res), () => {}, savedBody], - ['public comment POST', 'comment', 'comments', 'Comment', () => commentController.unProtectedPost(commentArgs(), res), () => {}, savedBody], + ['comment POST', 'comment', 'comments', 'Comment', () => commentController.protectedPost(commentArgs(), res), () => {}, savedBody], ['commentperiod PUT', 'commentPeriod', 'commentperiods', 'CommentPeriod', () => commentPeriodController.protectedPut(cpArgs(), res), () => {}, () => ({ ...writeResult })], ['project DELETE', 'project', 'projects', 'Project', () => projectController.protectedDelete(projArgs(), res), @@ -124,7 +124,7 @@ describe('DEMI push awaited before the reply', () => { // A failed push must not lead to a second insert: the record and its stored file exist once. [ ['document POST', 'document', 'documents', 'Document', () => documentController.protectedPost(docArgs(), res)], - ['public comment POST', 'comment', 'comments', 'Comment', () => commentController.unProtectedPost(commentArgs(), res)] + ['comment POST', 'comment', 'comments', 'Comment', () => commentController.protectedPost(commentArgs(), res)] ].forEach(([label, kind, route, modelName, run]) => { it(`${label} saves once and only flags the row when the push fails`, async () => { const save = sinon.spy(models[modelName].prototype, 'save'); diff --git a/test/controllers/demi-push-callsites.test.js b/test/controllers/demi-push-callsites.test.js index 59cf4508..a3cdce54 100644 --- a/test/controllers/demi-push-callsites.test.js +++ b/test/controllers/demi-push-callsites.test.js @@ -67,7 +67,6 @@ describe('DEMI push call sites', () => { afterEach(() => sinon.restore()); [ - ['document', documentController, 'unProtectedPost', docArgs], ['document', documentController, 'protectedPost', docArgs], ['document', documentController, 'protectedPut', docArgs], ['document', documentController, 'protectedPublish', docArgs], @@ -84,7 +83,6 @@ describe('DEMI push call sites', () => { ['commentPeriod', commentPeriodController, 'protectedPublish', cpArgs], ['commentPeriod', commentPeriodController, 'protectedUnPublish', cpArgs], ['comment', commentController, 'protectedPost', commentArgs], - ['comment', commentController, 'unProtectedPost', commentArgs], ['organization', organizationController, 'protectedPost', orgArgs], ['organization', organizationController, 'protectedPut', orgArgs], ['organization', organizationController, 'protectedPublish', orgArgs], diff --git a/test/controllers/document-upload.test.js b/test/controllers/document-upload.test.js index 24a760a2..7160d744 100644 --- a/test/controllers/document-upload.test.js +++ b/test/controllers/document-upload.test.js @@ -1,7 +1,7 @@ /** * Admin document upload (POST /document): images uploaded from the Update form (documentSource - * 'UPDATE') must be a small web image; every other source keeps the old limits. Both admin and - * public uploads record the sha256 of the uploaded bytes. + * 'UPDATE') must be a small web image; every other source keeps the old limits. An upload records + * the sha256 of the uploaded bytes, and a failed save removes the stored file. */ 'use strict'; @@ -32,6 +32,8 @@ const HEADERS = { describe('Document upload limits by source', () => { let res; let stored; + let saveError; + let writtenDespiteError; function uploadArgs(documentSource, upfile, overrides = {}) { const params = { @@ -52,13 +54,17 @@ describe('Document upload limits by source', () => { beforeEach(() => { res = { status: sinon.stub().returnsThis(), json: sinon.stub() }; stored = null; + saveError = null; + writtenDespiteError = false; function MockDocument() { - this.save = () => { stored = this; return Promise.resolve(this); }; + this._id = new mongoose.Types.ObjectId(); + this.save = () => saveError ? Promise.reject(saveError) : (stored = this, Promise.resolve(this)); } MockDocument.updateOne = () => Promise.resolve(); + MockDocument.exists = () => Promise.resolve(writtenDespiteError ? { _id: 'x' } : null); sinon.stub(mongoose, 'model').returns(MockDocument); sinon.stub(MinioController, 'putDocument').resolves({ path: 'minio/file', extension: 'png' }); - sinon.stub(MinioController, 'deleteDocument').resolves(); + sinon.stub(MinioController, 'deleteObject').resolves(); sinon.stub(fs, 'writeFileSync'); sinon.stub(fs, 'unlinkSync'); sinon.stub(Utils, 'recordAction').resolves(); @@ -186,19 +192,85 @@ describe('Document upload limits by source', () => { expect(stored.internalOriginalSha256).to.equal(ABC_SHA256); }); - it('records the sha256 of the bytes a member of the public uploads', async () => { - const args = { - swagger: { params: { - project: { value: PROJECT_ID }, - _comment: { value: '5f4c7d1e2b3a4c5d6e7f8092' }, - upfile: { value: { mimetype: 'application/pdf', originalname: 'letter.pdf', size: 3, buffer: Buffer.from('abc') } } - } }, - body: { documentAuthor: 'A. Person', documentAuthorType: '5f4c7d1e2b3a4c5d6e7f8093' } - }; + it('deletes the stored file by its full key when the document save fails', async () => { + const key = `${PROJECT_ID}/0a1b2c3d.pdf`; + MinioController.putDocument.resolves({ path: key, extension: 'pdf' }); + saveError = new Error('save failed'); - await documentController.unProtectedPost(args, res); + await documentController.protectedPost(uploadArgs('PROJECT', + { mimetype: 'application/pdf', originalname: 'report.pdf', size: 3, buffer: Buffer.from('abc') }), res); - expect(res.status.calledWith(200)).to.be.true; - expect(stored.internalOriginalSha256).to.equal(ABC_SHA256); + expect(res.status.calledWith(400)).to.be.true; + expect(MinioController.deleteObject.args).to.deep.equal([[MinioController.BUCKETS.DOCUMENTS_BUCKET, key]]); + }); + + it('keeps the stored file when the document was written despite the save error', async () => { + saveError = new Error('write timeout'); + writtenDespiteError = true; + + await documentController.protectedPost(uploadArgs('PROJECT', + { mimetype: 'application/pdf', originalname: 'report.pdf', size: 3, buffer: Buffer.from('abc') }), res); + + expect(res.status.args).to.deep.equal([[400]]); + expect(MinioController.deleteObject.called, 'stored file deleted').to.be.false; + }); + + it('still answers with the save error when the file cleanup also fails, and leaves no rejection unhandled', async () => { + saveError = new Error('save failed'); + MinioController.deleteObject.rejects(new Error('minio down')); + const unhandled = []; + const onUnhandled = reason => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + + try { + await documentController.protectedPost(uploadArgs('PROJECT', + { mimetype: 'application/pdf', originalname: 'report.pdf', size: 3, buffer: Buffer.from('abc') }), res); + await new Promise(resolve => setImmediate(resolve)); + } finally { + process.removeListener('unhandledRejection', onUnhandled); + } + + expect(res.status.args).to.deep.equal([[400]]); + expect(res.json.firstCall.args[0]).to.equal(saveError); + expect(unhandled).to.be.empty; + }); + + it('removes the temp file under uploads/ when the object storage write fails', async () => { + MinioController.putDocument.rejects(new Error('minio down')); + + await documentController.protectedPost(uploadArgs('PROJECT', + { mimetype: 'application/pdf', originalname: 'report.pdf', size: 3, buffer: Buffer.from('abc') }), res); + + expect(res.status.args).to.deep.equal([[400]]); + expect(fs.unlinkSync.args).to.deep.equal([[fs.writeFileSync.firstCall.args[0]]]); + }); + + const badIds = { + 'a project sent as an object': { project: { $ne: null } }, + 'a 12-character project id': { project: 'abcdefghijkl' }, + 'a comment id that is not hex': { _comment: 'zzzzzzzzzzzzzzzzzzzzzzzz' }, + 'a milestone sent as an object': { milestone: { _bsontype: 'ObjectId' } }, + 'a type id with extra characters': { type: `${PROJECT_ID}0` }, + 'the string "null" as the project': { project: 'null' }, + 'the string "null" as the comment id': { _comment: 'null' } + }; + + Object.entries(badIds).forEach(([label, overrides]) => { + it(`refuses ${label} before writing anything`, async () => { + await documentController.protectedPost(uploadArgs('PROJECT', + { mimetype: 'application/pdf', originalname: 'report.pdf', size: 3, buffer: Buffer.from('abc') }, overrides), res); + + expect(res.status.args).to.deep.equal([[400]]); + expect(fs.writeFileSync.called, 'temp file written').to.be.false; + expect(MinioController.putDocument.called, 'object storage written').to.be.false; + }); + }); + + it('accepts the string "null" the admin form sends for an empty milestone', async () => { + await documentController.protectedPost(uploadArgs('PROJECT', + { mimetype: 'application/pdf', originalname: 'report.pdf', size: 3, buffer: Buffer.from('abc') }, { milestone: 'null' }), res); + + expect(res.status.args).to.deep.equal([[200]]); + expect(stored.milestone).to.be.null; }); }); diff --git a/test/helpers/publicWriteRoutesRemoved.test.js b/test/helpers/publicWriteRoutesRemoved.test.js new file mode 100644 index 00000000..9495b65e --- /dev/null +++ b/test/helpers/publicWriteRoutesRemoved.test.js @@ -0,0 +1,56 @@ +/** + * The public site no longer takes comments or attachments, so POST /public/comment and + * POST /public/document are gone. Drives requests through the real swagger and router: a POST to + * either path must not reach a controller, the database or object storage. + */ +const { expect } = require('chai'); +const sinon = require('sinon'); +const express = require('express'); +const fs = require('fs'); +const mongoose = require('mongoose'); +const path = require('path'); +const request = require('supertest'); +const YAML = require('js-yaml'); + +const createRouter = require('../../api/middleware/swagger-router'); +const MinioController = require('../../api/helpers/minio'); + +const OID = '5f4c7d1e2b3a4c5d6e7f8091'; + +describe('public write routes removed', () => { + let app; + + before(() => { + const spec = YAML.load( + fs.readFileSync(path.join(__dirname, '../../api/swagger/swagger.yaml'), 'utf8') + ); + app = express(); + app.use(express.json()); + app.use('/api', createRouter(spec, [path.join(__dirname, '../../api/controllers')])); + }); + + beforeEach(() => { + sinon.stub(mongoose, 'model').throws(new Error('reached the database')); + sinon.stub(MinioController, 'putDocument').rejects(new Error('reached object storage')); + }); + + afterEach(() => sinon.restore()); + + it('does not route POST /public/comment', async () => { + const res = await request(app).post('/api/public/comment').send({ period: OID, comment: 'text', author: 'A' }); + + expect(res.status).to.equal(404); + expect(mongoose.model.called, 'reached the database').to.be.false; + }); + + it('does not route POST /public/document', async () => { + const res = await request(app).post('/api/public/document') + .field('project', OID) + .field('_comment', OID) + .attach('upfile', Buffer.from('%PDF-1.4'), { filename: 'letter.pdf', contentType: 'application/pdf' }); + + expect(res.status).to.equal(404); + expect(mongoose.model.called, 'reached the database').to.be.false; + expect(MinioController.putDocument.called, 'reached object storage').to.be.false; + }); +}); diff --git a/test/support/demiPushHarness.js b/test/support/demiPushHarness.js index bcdc535d..31517c91 100644 --- a/test/support/demiPushHarness.js +++ b/test/support/demiPushHarness.js @@ -94,10 +94,10 @@ function stubModel(modelName, saved) { const M = function (init) { Object.assign(this, init || {}); this.legislationYearList = this.legislationYearList || []; }; M.modelName = modelName; M.prototype.save = () => Promise.resolve(saved()); - // dateCompleted keeps comment.unProtectedPost inside the period; read[] is what publish toggles. + // read[] is what publish toggles. // project.protectedPublish only switches to a year the project lists and has a named block for. const stored = () => new M({ - _id: OID, project: OID, read: [], dateCompleted: new Date(Date.now() + 86400000), + _id: OID, project: OID, read: [], legislation_2002: { name: 'X', phaseHistory: '' }, currentLegislationYear: 'legislation_2002', legislationYearList: [2002] }); // recentActivity.protectedPut reads the stored row with .lean() before validating the merge