From ebb352d9e04599d2585a36d8c136ec84e8cb50ba Mon Sep 17 00:00:00 2001 From: Andres Pinto <143480783+apsantiso@users.noreply.github.com> Date: Tue, 7 Apr 2026 09:54:45 +0200 Subject: [PATCH 1/2] refactor(users-buckets): remove users and bucket model external dependency --- lib/core/users/MongoDBUsersRepository.ts | 25 +-- lib/core/users/User.ts | 1 - lib/engine.js | 11 +- lib/models/bucket.ts | 117 +++++++++++++ lib/models/database.js | 117 +++++++++++++ lib/models/user.ts | 200 +++++++++++++++++++++++ lib/server/middleware/authenticate.js | 20 +-- lib/server/routes/users.js | 33 ++-- tests/lib/e2e/storage-db-manager.ts | 9 +- tests/lib/e2e/utils.ts | 44 +++-- 10 files changed, 488 insertions(+), 89 deletions(-) create mode 100644 lib/models/bucket.ts create mode 100644 lib/models/database.js create mode 100644 lib/models/user.ts diff --git a/lib/core/users/MongoDBUsersRepository.ts b/lib/core/users/MongoDBUsersRepository.ts index 70504d38..7c41e1ae 100644 --- a/lib/core/users/MongoDBUsersRepository.ts +++ b/lib/core/users/MongoDBUsersRepository.ts @@ -13,7 +13,6 @@ type DatabaseUser = { }; totalUsedSpaceBytes: 0; maxSpaceBytes: 0; - referralPartner: null; subscriptionPlan: { isSubscribed: boolean; }; @@ -71,36 +70,20 @@ export class MongoDBUsersRepository implements UsersRepository { } async create(data: CreateUserData): Promise { - const user = await new Promise( - (resolve: (newUser: BasicUser) => void, reject) => { - this.userModel.create(data, (err: Error, user: DatabaseUser) => { - if (err) { - reject(err); - } else { - resolve({ - id: user.id, - maxSpaceBytes: user.maxSpaceBytes, - uuid: user.uuid, - }); - } - }); - } - ); + const createdUser = await this.userModel.create(data); // TODO: Change storage-models to insert only, avoiding updates. await this.userModel.updateOne( - { - _id: user.id, - }, + { _id: createdUser.id }, { maxSpaceBytes: data.maxSpaceBytes, activated: data.activated, } ); - user.maxSpaceBytes = data.maxSpaceBytes; + createdUser.maxSpaceBytes = data.maxSpaceBytes; - return user; + return createdUser; } async updateById(id: string, update: any): Promise { diff --git a/lib/core/users/User.ts b/lib/core/users/User.ts index be537571..b6867359 100644 --- a/lib/core/users/User.ts +++ b/lib/core/users/User.ts @@ -15,7 +15,6 @@ export interface User { subscriptionPlan?: { isSubscribed?: boolean; }; - referralPartner?: string | null; preferences?: { dnt: boolean; }; diff --git a/lib/engine.js b/lib/engine.js index 2c569a51..f2eb2f22 100644 --- a/lib/engine.js +++ b/lib/engine.js @@ -9,7 +9,7 @@ const express = require('express'); const crossorigin = require('cors'); const helmet = require('helmet'); const Config = require('./config'); -const Storage = require('storj-service-storage-models'); +const DatabaseConnection = require('./models/database'); const { querystring } = require('./server/middleware/query-string'); const { ErrorHandlerFactory: errorHandler } = require('./server/middleware/error-handler'); @@ -87,11 +87,7 @@ Engine.HEALTH_INTERVAL = 30000; Engine.prototype.start = function (callback) { log.info('starting the bridge engine'); - this.storage = new Storage( - this._config.storage.mongoUrl, - this._config.storage.mongoOpts, - { logger: log } - ); + this.storage = DatabaseConnection.createFromConfig(this._config.storage, log); const { QUEUE_USERNAME, QUEUE_PASSWORD, QUEUE_HOST } = this._config; @@ -127,7 +123,7 @@ Engine.prototype.start = function (callback) { this.mailer = new Mailer(this._config.mailer); this.contracts = new storj.StorageManager( - new MongoDBStorageAdapter(this.storage), + new MongoDBStorageAdapter(this.storage.connection), { disableReaper: true } ); this.redis = require('redis').createClient(this._config.redis); @@ -257,7 +253,6 @@ Engine.prototype._configureApp = function () { routers.forEach(bindRoute); app.use(unexpectedErrorLogger); app.use(errorHandler({ logger: log })); - app.use(json()); const profile = this._config.server.public || this._config.server; diff --git a/lib/models/bucket.ts b/lib/models/bucket.ts new file mode 100644 index 00000000..8db590a2 --- /dev/null +++ b/lib/models/bucket.ts @@ -0,0 +1,117 @@ +import crypto from "crypto"; +import { Schema, Document, Connection, Types } from "mongoose"; +import { validate as uuidValidate, version as uuidVersion } from "uuid"; +const errors = require("storj-service-error-types"); + +interface IBucket extends Document { + storage: number; + transfer: number; + status: "Active" | "Inactive"; + pubkeys: string[]; + user: string; + userId: string; + name: string; + maxFrameSize: number; + created: Date; + publicPermissions: ("PUSH" | "PULL")[]; + encryptionKey: string; +} + +const BucketSchema = new Schema( + { + storage: { type: Number, default: 0 }, + transfer: { type: Number, default: 0 }, + status: { + type: String, + enum: ["Active", "Inactive"], + default: "Active", + }, + pubkeys: [{ type: String, ref: "PublicKey" }], + user: { type: String, ref: "User" }, + userId: { + type: String, + required: true, + validate: { + validator: (value: string) => + uuidValidate(value) && uuidVersion(value) === 4, + message: "Invalid UUID", + }, + ref: "User", + }, + name: { + type: String, + default: () => "Bucket-" + crypto.randomBytes(3).toString("hex"), + }, + maxFrameSize: { type: Number, default: -1 }, + created: { type: Date, default: Date.now }, + publicPermissions: { + type: [{ type: String, enum: ["PUSH", "PULL"] }], + default: [], + }, + encryptionKey: { type: String, default: "" }, + }, + { + statics: { + async create( + user: { _id: string; uuid: string }, + data: { pubkeys?: string[]; name?: string }, + callback: (err: Error | null, bucket?: IBucket) => void, + ) { + const Bucket = this; + + const bucket = new Bucket({ + status: "Active", + pubkeys: data.pubkeys, + user: user._id, + userId: user.uuid, + }); + + if (data.name) { + bucket.name = data.name; + } + + try { + await bucket.save(); + + const savedBucket = await Bucket.findOne({ + _id: bucket._id, + }); + if (!savedBucket) { + return callback( + new errors.InternalError( + "Failed to load created bucket", + ), + ); + } + + callback(null, savedBucket); + } catch (err: any) { + if (err.code === 11000) { + return callback( + new errors.ConflictError( + "Name already used by another bucket", + ), + ); + } + callback(new errors.InternalError(err.message)); + } + }, + }, + }, +); + +BucketSchema.index({ user: 1 }); +BucketSchema.index({ userId: 1 }); +BucketSchema.index({ created: 1 }); +BucketSchema.index({ user: 1, name: 1 }, { unique: true }); + +BucketSchema.set("toObject", { + transform: (doc: any, ret: Record) => { + delete ret.__v; + delete ret._id; + ret.id = doc._id; + }, +}); + +export = (connection: Connection) => + connection.model("Bucket", BucketSchema); diff --git a/lib/models/database.js b/lib/models/database.js new file mode 100644 index 00000000..f5d8b17c --- /dev/null +++ b/lib/models/database.js @@ -0,0 +1,117 @@ +'use strict'; + +const assert = require('assert'); +const mongoose = require('mongoose'); + +/** + * MongoDB storage interface + * @constructor + * @param {String} mongoURI + * @param {Object} mongoOptions + * @param {Object} storageOptions + */ +function Database(mongoURI, mongoOptions, storageOptions) { + if (!(this instanceof Database)) { + return new Database(mongoURI, mongoOptions, storageOptions); + } + + assert(typeof mongoOptions === 'object', 'Invalid mongo options supplied'); + + this._uri = mongoURI; + this._options = mongoOptions; + this._log = (storageOptions && storageOptions.logger) || { + info: console.log, + debug: console.log, + error: console.error, + warn: console.warn, + }; + + this._connect(); +} + +Database.externalModels = require('storj-service-storage-models').models; +Database.localModels = { + Bucket: require('./bucket'), + User: require('./user'), +}; +Database.constants = require('../constants'); + +/** + * Connects to the database + */ +Database.prototype._connect = function () { + const opts = Object.assign({ ssl: false }, this._options); + + if (opts.server) { + this._log.warn( + 'Deprecated \'server\' option detected in database configuration. ' + + 'This option was removed in MongoDB driver 4.x and will be ignored. ' + + 'Please remove it from your configuration.' + ); + delete opts.server; + } + + this._log.info('opening database connection at %s', this._uri); + + this.connection = mongoose.createConnection(this._uri, opts); + + this.connection.on('error', (err) => { + this._log.error('database connection error: %s', err.message); + }); + + this.connection.on('disconnected', () => { + this._log.warn('disconnected from database'); + }); + + this.connection.on('connected', () => { + this._log.info('connected to database'); + }); + + this.models = this._createBoundModels(); +}; + +/** + * Return a dictionary of models bound to this connection + */ +Database.prototype._createBoundModels = function () { + const bound = {}; + + const allModels = { + ...Database.externalModels, + ...Database.localModels, + }; + + for (const model in allModels) { + bound[model] = allModels[model](this.connection); + } + + return bound; +}; + +/** + * Returns a promise that resolves when the connection is ready + */ +Database.prototype.ready = function () { + return new Promise((resolve, reject) => { + if (this.connection.readyState === 1) { + return resolve(); + } + this.connection.once('connected', resolve); + this.connection.once('error', reject); + }); +}; + +/** + * Creates a Database instance from a config object + * @param {Object} storageConfig - { mongoUrl, mongoOpts } + * @param {Object} [logger] + */ +Database.createFromConfig = function (storageConfig, logger) { + return new Database( + storageConfig.mongoUrl, + storageConfig.mongoOpts, + logger ? { logger } : {} + ); +}; + +module.exports = Database; diff --git a/lib/models/user.ts b/lib/models/user.ts new file mode 100644 index 00000000..07bea33f --- /dev/null +++ b/lib/models/user.ts @@ -0,0 +1,200 @@ +import crypto from "crypto"; +import { Schema, Document, Connection, Types } from "mongoose"; +import { + v4 as uuidv4, + validate as uuidValidate, + version as uuidVersion, +} from "uuid"; + +const errors = require("storj-service-error-types"); +const activator = require("hat").rack(256); +// NB: emails must conform to RFC 3969 (https://tools.ietf.org/html/rfc3696) +const isValidEmail = (email: string): boolean => + /^(.{1,64}@.{1,255}|.{1,255}@heroku\.storj\.io)$/.test(email); + +interface IBytesTracker { + lastHourStarted?: Date; + lastHourBytes: number; + lastDayStarted?: Date; + lastDayBytes: number; + lastMonthStarted?: Date; + lastMonthBytes: number; +} + +interface IUser extends Document { + _id: string; + uuid: string; + email: string; + hashpass: string | null; + pendingHashPass: string | null; + created: Date; + activator: unknown; + deactivator: unknown; + resetter: unknown; + activated: boolean; + isFreeTier: boolean; + preferences: { dnt: boolean }; + configuration: { + disableDeactivation: boolean; + disableResetPassword: boolean; + disableBucketDeletion: boolean; + }; + totalUsedSpaceBytes: number; + maxSpaceBytes: number; + activate(callback: (err?: Error) => void): Promise; + deactivate(callback: (err?: Error) => void): Promise; +} + +const UserSchema = new Schema( + { + _id: { + type: String, + required: true, + default: () => uuidv4(), + validate: { + validator: (value: string) => + (uuidValidate(value) && uuidVersion(value) === 4) || + isValidEmail(value), + message: "Invalid UUID", + }, + }, + uuid: { + type: String, + required: true, + default: () => uuidv4(), + validate: { + validator: (value: string) => + uuidValidate(value) && uuidVersion(value) === 4, + message: "Invalid UUID", + }, + }, + email: { + type: String, + required: true, + validate: { + validator: (value: string) => isValidEmail(value), + message: "Invalid user email address", + }, + }, + hashpass: { + type: String, + validate: { + validator: (value: string | null) => + value === null ? true : value.length === 64, + message: + "{VALUE} must either be 64 characters in length or null", + }, + }, + pendingHashPass: { type: String, default: null }, + created: { type: Date, default: Date.now }, + activator: { type: Schema.Types.Mixed, default: activator }, + deactivator: { type: Schema.Types.Mixed, default: null }, + resetter: { type: Schema.Types.Mixed, default: null }, + activated: { type: Boolean, default: false }, + isFreeTier: { type: Boolean, default: true }, + preferences: { + dnt: { type: Boolean, default: false, required: true }, + }, + totalUsedSpaceBytes: { type: Number, default: 0 }, + maxSpaceBytes: { type: Number, default: 0 }, + }, + { + statics: { + // TODO: The model is enforcing domain and bussiness logic, move this to a usecase or service at least the validation part. + async lookup(email: string, passwd: string): Promise { + const User = this; + if (!passwd) { + throw new errors.NotAuthorizedError("Invalid email or password"); + } + + const user = await User.findOne({ + email, + hashpass: crypto + .createHash("sha256") + .update(passwd) + .digest("hex"), + }); + + if (!user) { + throw new errors.NotAuthorizedError("Invalid email or password"); + } + + return user; + }, + // TODO: The model is enforcing domain and bussiness logic, move this to a usecase or service at least the validation part. + async create(opts: { + email: string; + password: string; + maxSpaceBytes: number; + activated: boolean; + }): Promise { + const User = this; + + if (!opts.email) { + throw new errors.BadRequestError("Must supply an email"); + } + + if ( + !opts.password || + Buffer.from(opts.password, "hex").length * 8 !== 256 + ) { + throw new errors.BadRequestError( + "Password must be hex encoded SHA-256 hash", + ); + } + + if (!isValidEmail(opts.email)) { + throw new errors.BadRequestError("Invalid email"); + } + + const existing = await User.findOne({ email: opts.email }); + if (existing) { + throw new errors.BadRequestError("Email address already registered"); + } + + const userUuid = uuidv4(); + const user = new User({ + _id: userUuid, + uuid: userUuid, + email: opts.email, + hashpass: crypto + .createHash("sha256") + .update(opts.password) + .digest("hex"), + maxSpaceBytes: opts.maxSpaceBytes, + activated: opts.activated, + }); + + await user.save(); + return user; + }, + }, + }, +); + +UserSchema.index({ resetter: 1 }); + +UserSchema.set("toJSON", { + virtuals: true, + transform: (doc: any, ret: Record) => { + delete ret.__v; + delete ret._id; + delete ret.pendingHashPass; + delete ret.bytesDownloaded; + delete ret.bytesUploaded; + }, +}); + +UserSchema.set("toObject", { + virtuals: true, + transform: (doc: any, ret: Record) => { + delete ret.__v; + delete ret._id; + delete ret.pendingHashPass; + delete ret.bytesDownloaded; + delete ret.bytesUploaded; + }, +}); + +export = (connection: Connection) => + connection.model("User", UserSchema); diff --git a/lib/server/middleware/authenticate.js b/lib/server/middleware/authenticate.js index fda19cc7..51fa1dfd 100644 --- a/lib/server/middleware/authenticate.js +++ b/lib/server/middleware/authenticate.js @@ -36,23 +36,21 @@ function AuthenticateMiddlewareFactory({ User, PublicKey, UserNonce }) { return [rawBodyMiddleware, authenticate]; } -AuthenticateMiddlewareFactory._basic = function ({ User }, req, res, next) { - let creds = basicauth(req); - User.lookup(creds.name, creds.pass, function (err, user) { - if (err) { - return next(err); - } +AuthenticateMiddlewareFactory._basic = async function ({ User }, req, res, next) { + const creds = basicauth(req); + + try { + const user = await User.lookup(creds.name, creds.pass); if (!user.activated) { - return next(errors.ForbiddenError( - 'User account has not been activated' - )); + return next(errors.ForbiddenError('User account has not been activated')); } req.user = user; - next(); - }); + } catch (err) { + return next(err); + } }; AuthenticateMiddlewareFactory._ecdsa = async function ({ User, PublicKey, UserNonce }, req, res, next) { diff --git a/lib/server/routes/users.js b/lib/server/routes/users.js index cec673fe..adbe85d6 100644 --- a/lib/server/routes/users.js +++ b/lib/server/routes/users.js @@ -168,26 +168,19 @@ UsersRouter.prototype.createUser = async function (req, res, next) { // Register log.debug('registering user account for %s', req.body.email); - if (req.body.referralPartner) { - self._createUserWithOpts(req, res, next); - } else { - User.create(req.body.email, req.body.password, async function (err, user) { - if (err) { - return next(err); - } - - user.maxSpaceBytes = 1024 * 1024 * 1024 * 2; - user.activated = true; - - trackUserActivated(user.uuid, req.body.email); - try { - await user.save(); - self._dispatchAndCreatePubKey(user, req, res, next); - } catch (err) { - return next(err); - } + try { + const user = await User.create({ + email: req.body.email, + password: req.body.password, + maxSpaceBytes: 1024 * 1024 * 1024 * 2, + activated: true, }); + + trackUserActivated(user.uuid, req.body.email); + self._dispatchAndCreatePubKey(user, req, res, next); + } catch (err) { + return next(err); } }; @@ -397,7 +390,7 @@ UsersRouter.prototype.destroyUser = async function (req, res, next) { }, }; - await sendGridMail.send(emailData, false).catch(err=>{ + await sendGridMail.send(emailData, false).catch(err => { log.error('[USERS/DESTROY_USER]: failed to send deactivation email, reason: %s', err.message); return next(new errors.InternalError(err.message)); @@ -481,7 +474,7 @@ UsersRouter.prototype.confirmDestroyUserStripe = async function (req, res, next) const stripeUser = await stripeUtils .customerExists(user._id) - .catch((err)=>{ + .catch((err) => { log.error( '[USERS/STRIPE_CONFIRM_DESTROY]: Cannot determine customer exists on stripe: %s, user email: %s', err.message, diff --git a/tests/lib/e2e/storage-db-manager.ts b/tests/lib/e2e/storage-db-manager.ts index cad0d678..8f51e7d0 100644 --- a/tests/lib/e2e/storage-db-manager.ts +++ b/tests/lib/e2e/storage-db-manager.ts @@ -35,14 +35,17 @@ export class StorageDbManager { this.connection = mongooseInstance.connection; - const modelFactories = require('storj-service-storage-models/lib/models'); + const externalModelFactories = require('storj-service-storage-models/lib/models'); + const localModelFactories = require('../../../lib/models/database').localModels; - Object.keys(modelFactories).forEach(modelName => { + const allModelFactories = { ...externalModelFactories, ...localModelFactories }; + + Object.keys(allModelFactories).forEach(modelName => { if (!this.models) { this.models = {} as Models; } - (this.models as any)[modelName] = modelFactories[modelName](this.connection); + (this.models as any)[modelName] = allModelFactories[modelName](this.connection); }); } diff --git a/tests/lib/e2e/utils.ts b/tests/lib/e2e/utils.ts index 9a9a48c1..4fcf5182 100644 --- a/tests/lib/e2e/utils.ts +++ b/tests/lib/e2e/utils.ts @@ -6,27 +6,23 @@ import { engine, intervalRefs } from "./setup"; import { generateTestUserData, TestUser, User } from "./users.fixtures"; import { sign } from "jsonwebtoken"; -type Args = { storage?: any, user: TestUser } - -const createdUsers: User[] = [] -export const createTestUser = async (args: Partial = {}): Promise => { - const { storage = engine.storage, user = generateTestUserData() } = args - const payload = { email: user.email, password: user.password } - const createdUser: User = await new Promise((resolve, reject) => storage.models.User.create(payload, (err: Error, user: any) => { - err ? reject(err) : resolve(user.toObject()) - })) - - await storage.models.User.updateOne( - { uuid: createdUser.uuid, }, - { maxSpaceBytes: user.maxSpaceBytes, activated: true, } - ); - - createdUser.maxSpaceBytes = user.maxSpaceBytes - createdUser.activated = true; - createdUser.password = user.password - - createdUsers.push(createdUser) - return createdUser +type Args = { storage?: any; user: TestUser }; + +const createdUsers: User[] = []; +export const createTestUser = async ( + args: Partial = {}, +): Promise => { + const { storage = engine.storage, user = generateTestUserData() } = args; + const created = await storage.models.User.create({ + email: user.email, + password: user.password, + maxSpaceBytes: user.maxSpaceBytes, + activated: true, + }); + const createdUser: User = { ...created.toObject(), password: user.password }; + + createdUsers.push(createdUser); + return createdUser; } export const cleanUpTestUsers = async (): Promise => { @@ -41,11 +37,9 @@ export const deleteTestUser = async (args: Args): Promise => { export const getAuth = (user: Omit) => { const credential = Buffer.from(`${user.email}:${user.password}`).toString('base64'); return `Basic ${credential}`; -} - +}; export const shutdownEngine = async () => { - await Promise.all([ engine.storage.connection.close(), engine.networkQueue.close(), @@ -179,7 +173,7 @@ export async function getProofOfWork( challenge: string, target: string, startNonce = 0, - maxNonce = 1000000 + maxNonce = 1000000, ) { const scryptOpts = { N: Math.pow(2, 10), r: 1, p: 1 }; From a19d8e6790b97284847ee931fb20bca7c324975a Mon Sep 17 00:00:00 2001 From: Andres Pinto <143480783+apsantiso@users.noreply.github.com> Date: Wed, 8 Apr 2026 13:04:31 +0200 Subject: [PATCH 2/2] fix: set free users space to 1GB --- lib/server/routes/users.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server/routes/users.js b/lib/server/routes/users.js index adbe85d6..21838b93 100644 --- a/lib/server/routes/users.js +++ b/lib/server/routes/users.js @@ -173,7 +173,7 @@ UsersRouter.prototype.createUser = async function (req, res, next) { const user = await User.create({ email: req.body.email, password: req.body.password, - maxSpaceBytes: 1024 * 1024 * 1024 * 2, + maxSpaceBytes: 1024 * 1024 * 1024 * 1, activated: true, });