From 2974bd3c5e63369fdc54a2eef0854768c438401b Mon Sep 17 00:00:00 2001 From: 7bit <662932+sevenbitbyte@users.noreply.github.com> Date: Wed, 25 Mar 2026 12:55:38 -0700 Subject: [PATCH 01/50] Update match-maker-client.js --- src/party/peer/match-maker-client.js | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 3ebd7fa..c5c3791 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -12,7 +12,7 @@ const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws'){ + constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws', billingIdentity=null){ super() @@ -22,6 +22,7 @@ class MatchMakerClient extends EventEmitter { this.identity = identity this.wsParty = null this.restParty = null + this.billingIdentity = null if(typeof urlOrParty == 'string'){ this.restUrl = urlOrParty @@ -157,15 +158,23 @@ class MatchMakerClient extends EventEmitter { debug('calling onInviteMsg') await pending.onInviteMsg(msg.invite) - + } } + async announceBillingKey({stripeCheckoutSession}={}){ + this.announcePublicKeys(true, { + stripe: stripeCheckoutSession + }) + } + async announcePublicKeys(useBillingKeyAsActor=false, billingMethodDetails=null){ - async announcePublicKeys(){ + let currentActor = useBillingKeyAsActor == true ? this.billingIdentity : this.identity + const announceData = { annoucement: { + type: useBillingKeyAsActor ? 'billing_identity' : 'user_identity' created: Date.now(), expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { @@ -174,9 +183,9 @@ class MatchMakerClient extends EventEmitter { public: this.sessionKey.key.public }, actorKey: { - type: this.identity.key.type, - hash: this.identity.key.hash, - public: this.identity.key.public + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public } }, trust: { @@ -185,8 +194,10 @@ class MatchMakerClient extends EventEmitter { } } + if( + - const actorSigMsg = await this.identity.sign(announceData.annoucement, true) + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) debug('actorSigMsg', actorSigMsg) From 34d7484c894fa3cf34ee79e77a441ceb90088b78 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 18 May 2026 20:14:04 +0000 Subject: [PATCH 02/50] add runner to EndpointContext --- src/service/endpoint-context.js | 7 ++++++- src/service/service-runner-node.js | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/service/endpoint-context.js b/src/service/endpoint-context.js index 80c725a..272a550 100644 --- a/src/service/endpoint-context.js +++ b/src/service/endpoint-context.js @@ -15,7 +15,7 @@ class EndpointContext { * @param {Debug} options.debug Debug constructor (defaults to npm:Debug) * @param {boolean} options.sendFullErrors Enables sending full stack traces to client (defaults to false) */ - constructor({party, endpoint, req, res, input, debug=Debug, sendFullErrors=false}){ + constructor({party, endpoint, runner, req, res, input, debug=Debug, sendFullErrors=false}){ /** * @member module:Service.EndpointContext.debug @@ -27,6 +27,11 @@ class EndpointContext { */ this.endpoint = endpoint + /** + * @member module:Service.EndpointContext.runner + */ + this.runner = runner + /** * @member module:Service.EndpointContext.MiddlewareConfig */ diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index 5df899c..b251c2e 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -516,6 +516,7 @@ class ServiceRunnerNode { req: event.request, res: event.response, endpoint, party: this.party, + runner: this, input: event.request.body, debug: Debug, sendFullErrors: this.sendFullErrors From 3b407d6981ef1049c64070cd03683bc360986ffd Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 18 May 2026 20:54:54 +0000 Subject: [PATCH 03/50] key management and ephemeral sessions --- package.json | 1 + src/party/peer/match-maker-client.js | 8 +- src/venue/endpoints/key-announce.js | 204 +++++++++++++++++++ src/venue/middleware/pre/ephmeral-session.js | 101 +++++++++ src/venue/schema/public-key.js | 47 +++++ src/venue/schema/session-key.js | 69 +++++++ src/venue/venue-service.js | 5 + 7 files changed, 431 insertions(+), 4 deletions(-) create mode 100644 src/venue/endpoints/key-announce.js create mode 100644 src/venue/middleware/pre/ephmeral-session.js create mode 100644 src/venue/schema/public-key.js create mode 100644 src/venue/schema/session-key.js diff --git a/package.json b/package.json index 5caa37b..4687a73 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "devDependencies": { "@dataparty/bouncer-model": "1.4.3", "@hapi/code": "^9.0.1", + "@hapi/joi": "^17.1.1", "@hapi/lab": "^25.0.1", "argon2": "^0.30.3", "argon2-browser": "^1.18.0", diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index c5c3791..65969a5 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -174,7 +174,7 @@ class MatchMakerClient extends EventEmitter { const announceData = { annoucement: { - type: useBillingKeyAsActor ? 'billing_identity' : 'user_identity' + //type: 'guest',//useBillingKeyAsActor ? 'billing_identity' : 'user_identity', created: Date.now(), expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { @@ -194,8 +194,6 @@ class MatchMakerClient extends EventEmitter { } } - if( - const actorSigMsg = await currentActor.sign(announceData.annoucement, true) const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) @@ -208,7 +206,9 @@ class MatchMakerClient extends EventEmitter { debug('announcePublicKeys', announceData) - const announceResult = await this.restParty.comms.call('key/announce', announceData, { + let callPath = useBillingKeyAsActor ? 'billing/key/announce' : 'key/announce' + + const announceResult = await this.restParty.comms.call(callPath, announceData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: false diff --git a/src/venue/endpoints/key-announce.js b/src/venue/endpoints/key-announce.js new file mode 100644 index 0000000..fca2d26 --- /dev/null +++ b/src/venue/endpoints/key-announce.js @@ -0,0 +1,204 @@ +const Joi = require('@hapi/joi') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('dataparty.endpoint.key-announce') + +const {Identity, Message, Routines} = require('@dataparty/crypto') + +//const IEndpoint = require('@dataparty/api/src/service/iendpoint') +const IEndpoint = require('../../service/iendpoint') + +const KeyVerifier = Joi.object().keys({ + id: Joi.string().max(100), + type: Joi.string().max(300).required(), + hash: Joi.string().max(200).required(), + public: { + box: Joi.string().max(60).required(), + sign: Joi.string().max(60).required(), + pqkem: Joi.string().max(8000).required(), + pqsign_ml: Joi.string().max(8000).required(), + pqsign_slh: Joi.string().max(800).required() + } +}) + +module.exports = class KeyAnnounceEndpoint extends IEndpoint { + + static get Name(){ + return 'key/announce' + } + + static get Description(){ + return 'announce a public key' + } + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + validate: Joi.object().keys({ + + annoucement: { + created: Joi.number().required(), + expiry: Joi.number().required(), + actorKey: KeyVerifier.required(), + sessionKey: KeyVerifier.required(), + }, + trust:{ + actorSig: Joi.string().required().description('actor signature of the annoucement in base64'), + sessionSig: Joi.string().required().description('session signature of the annoucement in base64') + } + + }).description('key to announce') + }, + post: { + encrypt: true, + validate: Joi.object().keys({ + done: Joi.boolean(), + }).description('public key') + } + } + } + + static async run(ctx, {Package}){ + + ctx.debug('hello key/announce') + + ctx.debug('ip', ctx.req.ip) + ctx.debug('input', ctx.input) + ctx.debug('sender', ctx.senderKey) + + const inputActorKey = { + type: ctx.input.annoucement.actorKey.type, + hash: ctx.input.annoucement.actorKey.hash, + public: ctx.input.annoucement.actorKey.public + } + + const inputSessionKey = { + type: ctx.input.annoucement.sessionKey.type, + hash: ctx.input.annoucement.sessionKey.hash, + public: ctx.input.annoucement.sessionKey.public + } + + + const computedActorHash = await Routines.hashKey( inputActorKey ) + const computedSessionHash = await Routines.hashKey( inputSessionKey ) + + ctx.debug('computed hash -', computedSessionHash) + + // verify keys are self consistent + if( + computedActorHash != inputActorKey.hash || + computedSessionHash != inputSessionKey.hash + ) { + ctx.debug('invalid actor or session key hash') + return {done: false} + } + + // ensure sender is connected using session key mentioned in annoucement + if(computedSessionHash == inputSessionKey.hash && + inputSessionKey.public.sign == ctx.senderKey.public.sign && + inputSessionKey.public.box == ctx.senderKey.public.box + ){ + + const actorSigBson = Routines.Utils.base64.decode( ctx.input.trust.actorSig ) + const sessionSigBson = Routines.Utils.base64.decode( ctx.input.trust.sessionSig ) + + const actorSigMsg = new Message({ msg: ctx.input.annoucement }) + const sessionSigMsg = new Message({ msg: ctx.input.annoucement }) + + actorSigMsg.sig = actorSigBson + sessionSigMsg.sig = sessionSigBson + + const actorIdentity = Identity.fromJSON({ + id: '', + key: inputActorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: '', + key: inputSessionKey + }) + + //verify actor & session signature. Require postquantum signing + await actorSigMsg.assertVerified( actorIdentity, true ) + await sessionSigMsg.assertVerified( sessionIdentity, true ) + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash').equals(computedSessionHash) + .exec())[0] + + if(!sessionKeyDoc){ + // create session document + + const now = Date.now() + const tomorrow = now + 24*60*60*1000 + + const fiveMinAgo = now - (5*1000*60) + const fiveMinFromNow = now + (5*1000*60) + + // verify start time is valid + if( + ctx.input.annoucement.created < fiveMinAgo || + ctx.input.annoucement.created > fiveMinFromNow + ) { + ctx.debug('invalid start time') + return {done: false} + } + + // verify expiry time is valid + if( + ctx.input.annoucement.expiry < now || + ctx.input.annoucement.expiry > tomorrow + ) { + ctx.debug('invalid expiry time') + return {done: false} + } + + ctx.debug('opening session -', computedSessionHash) + + let sessionDoc = await ctx.party.createDocument('session_key', { + created: now, + expiry: ctx.input.annoucement.expiry, + annoucement: ctx.input.annoucement, + trust: ctx.input.trust + }) + + ctx.debug('session created - ', computedSessionHash) + + } else { + ctx.debug('session key already known - ', computedSessionHash) + return {done: false} + } + + let publicKey = (await ctx.party.find() + .type('public_key') + .where('annoucement.actorKey.hash').equals(computedActorHash) + .exec())[0] + + if(!publicKey){ + + ctx.debug('annoucing key -', computedActorHash) + + let keyDoc = await ctx.party.createDocument('public_key', { + created: Date.now(), + role: 'guest', + owner: computedActorHash, + ...inputActorKey + }) + + ctx.debug('actor annouced key -', computedActorHash) + } else { + ctx.debug('key already known -', computedActorHash) + } + + return {done: true} + + } else { + + ctx.debug('announce ERROR - BAD KEY HASH') + + return {done: false} + } + + } +} diff --git a/src/venue/middleware/pre/ephmeral-session.js b/src/venue/middleware/pre/ephmeral-session.js new file mode 100644 index 0000000..11f862e --- /dev/null +++ b/src/venue/middleware/pre/ephmeral-session.js @@ -0,0 +1,101 @@ +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') + +const IMiddleware = require('../../service/imiddleware') + +module.exports = class Decrypt extends IMiddleware { + + static get Name(){ + return 'ephemeral_session' + } + + static get Type(){ + return 'pre' + } + + static get Description(){ + return 'Decrypt inbound data' + } + + static get ConfigSchema(){ + return Joi.boolean() + } + + static async start(party){ + + } + + static async run(ctx, {Config}){ + + if (!Config){ return } + + if(!ctx.input_session_id){ + throw new Error('no session id') + } + + ctx.debug('looking up session -', ctx.input_session_id) + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(ctx.input_session_id) + .exec() + )[0] + + if(!sessionKeyDoc){ + throw new Error('invalid session') + } + + ctx.debug('located session - ', ctx.input_session_id) + + const sessionKey = sessionKeyDoc.data.annoucement.sessionKey + const actorKey = sessionKeyDoc.data.annoucement.actorKey + + // ensure sender is connected using session key mentioned in db + if(sessionKey.hash == ctx.input_session_id && + sessionKey.public.sign == ctx.senderKey.public.sign && + sessionKey.public.box == ctx.senderKey.public.box + ){ + + const now = Date.now() + + if( sessionKeyDoc.data.expiry < now ){ + throw new Error('session expired!') + } + + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: actorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: 'ephemeral-session', + key: sessionKey + }) + + ctx.debug('looking up actor - ', actorKey.hash) + + let actorDoc = (await ctx.party.find() + .type('public_key') + .where('hash') + .equals(actorKey.hash) + .exec() + )[0] + + if(!actorDoc){ + throw new Error('failed to find actor') + } + + ctx.setSession( sessionKeyDoc ) + ctx.setIdentity( actorIdentity ) + ctx.setActor( actorDoc ) + + ctx.debug('session ready') + } else { + throw new Error('invalid sender key for this session') + } + + } +} \ No newline at end of file diff --git a/src/venue/schema/public-key.js b/src/venue/schema/public-key.js new file mode 100644 index 0000000..597601b --- /dev/null +++ b/src/venue/schema/public-key.js @@ -0,0 +1,47 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + +class PublicKey extends ISchema { + + static get Type () { return 'public_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + + role: String, // [ guest, billing, service, wallet ] + + owner: {required: true, index: true, type: String}, // public key hash + + type: String, + hash: {required: true, index: true, type: String, unique: true}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } + } + + static setupSchema(schema){ + //schema.index({ 'hash': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = PublicKey diff --git a/src/venue/schema/session-key.js b/src/venue/schema/session-key.js new file mode 100644 index 0000000..a0da93c --- /dev/null +++ b/src/venue/schema/session-key.js @@ -0,0 +1,69 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + + +function PublicKeySchema(unique=true){ + return { + type: {required: true, type: String}, + hash: {required: true, index: true, type: String, unique}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } +} + + +class SessionKey extends ISchema { + + static get Type () { return 'session_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + index: true, + required: true + }, + annoucement: { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + required: true + }, + sessionKey: PublicKeySchema(true), + actorKey: PublicKeySchema(false) + }, + trust: { + actorSig: {required: true, type: String}, //! base64 of BSON signature + sessionSig: {required: true, type: String} //! base64 of BSON signature + } + } + } + + static setupSchema(schema){ + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = SessionKey \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index f5c7ad3..47eaad7 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -11,6 +11,9 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) + + builder.addSchema(Path.join(__dirname, './schema/public-key.js')) + builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_service.js')) builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) @@ -22,6 +25,8 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(DatapartySrv.endpoint_paths.identity) builder.addEndpoint(DatapartySrv.endpoint_paths.version) + builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) builder.addAuth(Path.join(__dirname, './auth.js')) } From 6df2d7079c14b0cea040b80840acf7bc8eab8033 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 18 May 2026 21:07:08 +0000 Subject: [PATCH 04/50] cleanup ephemeral sessions every 15 minutes --- src/venue/tasks/cleanup-ephemeral-sessions.js | 86 +++++++++++++++++++ src/venue/venue-service.js | 4 + 2 files changed, 90 insertions(+) create mode 100644 src/venue/tasks/cleanup-ephemeral-sessions.js diff --git a/src/venue/tasks/cleanup-ephemeral-sessions.js b/src/venue/tasks/cleanup-ephemeral-sessions.js new file mode 100644 index 0000000..435a48b --- /dev/null +++ b/src/venue/tasks/cleanup-ephemeral-sessions.js @@ -0,0 +1,86 @@ +const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') + +//const ITask = require('@dataparty/api/src/service/itask') +const ITask = require('../../service/itask') + +class CleanupEphemeralSessionsTask extends ITask { + + constructor(options){ + super({ + name: CleanupEphemeralSessionsTask.name, + background: CleanupEphemeralSessionsTask.Config.background, + ...options + }) + + debug('new') + + this.duration = Math.round(1000*60*15) + this.timeout = null + } + + static get Config(){ + return { + background: true, + autostart: true + } + } + + async exec(){ + + this.setTimer() + + return this.detach() + } + + + async lookupSessions(){ + + let now = Date.now() + + return (await this.context.party.find() + .type('session_key') + .where('expiry').lt(now) + .exec()) + } + + setTimer(){ + this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) + } + + async onTimeout(){ + this.timeout = null + + debug('cleanup ephemeral sessions task') + + try{ + let sessions = await this.lookupSessions() + + if(sessions && sessions.length > 0){ + debug('expired sessions ', sessions.length) + let list = sessions.map(i=>{return i.data}) + await this.context.party.remove(...list) + } + } catch (err){ + debug(err) + } + + this.setTimer() + } + + stop(){ + if(this.timeout !== null){ + clearTimeout(this.timeout) + this.timeout = null + } + } + + static get Name(){ + return 'cleanup-ephemeral-sessions' + } + + static get Description(){ + return 'Cleanup Ephemeral sessions' + } +} + +module.exports = CleanupEphemeralSessionsTask \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index 47eaad7..dc56ea1 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -28,6 +28,10 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) + + + builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) + builder.addAuth(Path.join(__dirname, './auth.js')) } } From 8ddc422009b0ed69fa1f169887dce0e5e0232ef4 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 25 May 2026 17:13:08 +0000 Subject: [PATCH 05/50] admin only access to API --- src/comms/peer-comms.js | 7 ++-- src/config/json-file.js | 13 +++++++ src/party/peer/match-maker-client.js | 6 ++- src/service/service-host.js | 2 +- src/venue/auth.js | 32 ++++++++++++++-- src/venue/bin/add-admin.js | 40 ++++++++++++++++++++ src/venue/bin/chill.js | 0 src/venue/endpoints/key-announce.js | 16 +++++++- src/venue/public/p2p-test.html | 7 +++- src/venue/schema/venue_package.js | 56 ++++++++++++++++++++++++++++ src/venue/schema/venue_service.js | 1 - src/venue/venue-host.js | 12 +++--- 12 files changed, 176 insertions(+), 16 deletions(-) create mode 100755 src/venue/bin/add-admin.js create mode 100644 src/venue/bin/chill.js create mode 100644 src/venue/schema/venue_package.js diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index c19a07c..b64c98a 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -385,13 +385,13 @@ class PeerComms extends ISocketComms { if(this.party.hostRunner){ const actor = await this.party.hostRunner.auth.lookupIdentity(offer.sender) - const verified = await Routines.verifyDataPQ(actor, signature, offerBSON) + const verified = await Routines.verifyDataPQ(offer.sender, signature, offerBSON) if(!verified){ throw new Error('DENY(hostRunner) - auth op signature is not valid') } - if(this.discoverRemoteIdentity){ this.remoteIdentity = actor } + if(this.discoverRemoteIdentity){ this.remoteIdentity = offer.sender } const authorized = await this.party.hostRunner.auth.isSocketConnectionAllowed(actor) if(!authorized){ @@ -406,6 +406,7 @@ class PeerComms extends ISocketComms { await this.stop() debug('DENY - client not allowed - ', this.remoteIdentity) + throw new Error('DENY - client not allowed') } } else { const actor = offer.sender @@ -420,7 +421,7 @@ class PeerComms extends ISocketComms { } } - debug('clienr auth op offer -', offer) + debug('client auth op offer -', offer) debug('ALLOW - allowing client - ', this.remoteIdentity) this.aesStream = await AESStream.recoverStream( diff --git a/src/config/json-file.js b/src/config/json-file.js index b5280ab..f2f1bec 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -22,6 +22,7 @@ class JsonFileConfig extends IConfig { this.path = this.basePath +'/config.json' this.defaults = defaults || {} this.content = Object.assign({}, this.defaults) + this.writing = false } async load(){ @@ -49,6 +50,8 @@ class JsonFileConfig extends IConfig { async start () { await this.touchDir('') await this.load() + + fs.watchFile(this.path, this.handleFileChange.bind(this)) logger('started') } @@ -79,7 +82,9 @@ class JsonFileConfig extends IConfig { } async save(){ + this.writing = true fs.writeFileSync(this.path, JSON.stringify(this.content, null, 2)) + this.writing = false } async touchDir (path) { @@ -98,6 +103,14 @@ class JsonFileConfig extends IConfig { }) }) } + + async handleFileChange(current, previous){ + if(this.writing){ return } + + logger('config changed, reloading') + + await this.load() + } } module.exports = JsonFileConfig \ No newline at end of file diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 65969a5..cf2f6bd 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -83,7 +83,7 @@ class MatchMakerClient extends EventEmitter { await this.announcePublicKeys() } - if(!this.wsParty){ + if(!this.wsParty && this.wsUrl){ this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -213,6 +213,10 @@ class MatchMakerClient extends EventEmitter { sendClearTextRequest: false, useSessions: false }) + + if(announceResult.done != true){ + throw new Error('annoucement request failed - '+callPath) + } } diff --git a/src/service/service-host.js b/src/service/service-host.js index 6fd1c68..a03a5dd 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -307,7 +307,7 @@ class ServiceHost { this.apiApp.use((err, req, res, _next) => { console.log('Error handler', err) if (err instanceof IpDeniedError) { - //res.status(401) + res.status(401) } else { res.status(err.status || 500) } diff --git a/src/venue/auth.js b/src/venue/auth.js index c2ff01c..7b8074c 100644 --- a/src/venue/auth.js +++ b/src/venue/auth.js @@ -1,6 +1,7 @@ const debug = require('debug')('dataparty.auth.venue-auth') const IAuth = require('../service/iauth') +const {Identity} = require('@dataparty/crypto') module.exports = class IAuth { @@ -36,12 +37,28 @@ module.exports = class IAuth { } async lookupIdentity(identity){ + + let sessionKeyDoc = (await this.context.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(identity.key.hash) + .exec() + )[0] + + if(sessionKeyDoc){ + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: sessionKeyDoc.data.annoucement.actorKey + }) + + return actorIdentity + } + return identity } async isSocketConnectionAllowed(identity){ - //throw new Error('not implemented') - return true + return await this.isAdmin(identity) } async isInternal(identity){ @@ -49,7 +66,16 @@ module.exports = class IAuth { } async isAdmin(identity){ - return false + + // verify key-hash is an admin + const admins = (await this.context.party.config.read('admins')) || [] + + if(admins.indexOf(identity.key.hash) == -1){ + debug('non-admin user', identity.key.hash) + return false + } + + return true } async canReadDb(identity){ diff --git a/src/venue/bin/add-admin.js b/src/venue/bin/add-admin.js new file mode 100755 index 0000000..f09375e --- /dev/null +++ b/src/venue/bin/add-admin.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +const Dataparty = require('../../index') + + +async function main(){ + + const path = '/data/dataparty/venue-service' + + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + await config.start() + + console.log(process.argv) + + let admins = (await config.read('admins')) || [] + + const newAdmin = process.argv[2] + + console.log(await config.readAll()) + console.log(admins) + + if(admins.indexOf(newAdmin) != -1){ return } + + admins.push(newAdmin) + + await config.write('admins', admins) + + console.log('admin added -', newAdmin) + +} + + +main().catch(err=>{ + console.error(err) +}).finally(()=>{ + process.exit() +}) diff --git a/src/venue/bin/chill.js b/src/venue/bin/chill.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/endpoints/key-announce.js b/src/venue/endpoints/key-announce.js index fca2d26..de1e7ec 100644 --- a/src/venue/endpoints/key-announce.js +++ b/src/venue/endpoints/key-announce.js @@ -93,6 +93,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { return {done: false} } + // ensure sender is connected using session key mentioned in annoucement if(computedSessionHash == inputSessionKey.hash && inputSessionKey.public.sign == ctx.senderKey.public.sign && @@ -122,6 +123,19 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { await actorSigMsg.assertVerified( actorIdentity, true ) await sessionSigMsg.assertVerified( sessionIdentity, true ) + // verify key-hash is an admin + //const admins = (await ctx.party.config.read('admins')) || [] + + //if(admins.indexOf(computedActorHash) == -1){ + const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) + if(!isAdmin){ + ctx.debug('non-admin user') + return {done: false} + + process.exit(1) + } + + let sessionKeyDoc = (await ctx.party.find() .type('session_key') .where('annoucement.sessionKey.hash').equals(computedSessionHash) @@ -172,7 +186,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { let publicKey = (await ctx.party.find() .type('public_key') - .where('annoucement.actorKey.hash').equals(computedActorHash) + .where('hash').equals(computedActorHash) .exec())[0] if(!publicKey){ diff --git a/src/venue/public/p2p-test.html b/src/venue/public/p2p-test.html index 6c57aa8..a45f222 100644 --- a/src/venue/public/p2p-test.html +++ b/src/venue/public/p2p-test.html @@ -256,7 +256,12 @@

Offers recieved

await hostLocal.start() - matchMaker = new Dataparty.MatchMakerClient(hostLocal.privateIdentity, null) + matchMaker = new Dataparty.MatchMakerClient( + hostLocal.privateIdentity, + null, + 'https://api.dataparty.xyz/api', + 'wss://api.dataparty.xyz/ws' + ) await matchMaker.start() diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js new file mode 100644 index 0000000..261f55e --- /dev/null +++ b/src/venue/schema/venue_package.js @@ -0,0 +1,56 @@ +'use strict' + +const debug = require('debug')('venue.venue_pkg') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenuePkg extends ISchema { + + static get Type () { return 'venue_pkg' } + + static get Schema(){ + return { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + venue: {type: String}, + settings: { + enabled: {type: Boolean, required: true}, + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + defaultConfig: {type: Object} + }, + package: { + name: {type: String, required: true, index: true}, + version: {type: String, required: true}, + githash: {type: String, required: true}, + branch: {type: String, required: true} + }, + compressedBuild: {type: String, required: true}, //! zlib compressed + signature: { + timestamp: {type: Number, required: true}, + type: {type: String, required: true, maxlength: 10}, + value: {type: String, required: true} + } + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenuePkg \ No newline at end of file diff --git a/src/venue/schema/venue_service.js b/src/venue/schema/venue_service.js index f5f8d08..f66ff9e 100644 --- a/src/venue/schema/venue_service.js +++ b/src/venue/schema/venue_service.js @@ -2,7 +2,6 @@ const debug = require('debug')('venue.venue_srv') - const ISchema = require('../../bouncer/ischema') const Utils = ISchema.Utils diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index fef4471..b3f8ab2 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -55,7 +55,8 @@ async function main(){ const CloudFlareIpFilter = { options: { - mode: 'allow' + mode: 'allow', + //trustProxy: true }, ips: [ '173.245.48.0/20', @@ -79,7 +80,8 @@ async function main(){ '2405:b500::/32', '2405:8100::/32', '2a06:98c0::/29', - '2c0f:f248::/32' + '2c0f:f248::/32', + '10.115.68.55/32' ] } @@ -114,7 +116,7 @@ async function main(){ trust_proxy: true, wsEnabled: true, ssl_key, ssl_cert, - listenUri: 'https://0.0.0.0:443', + listenUri: 'https://0.0.0.0:3000', staticPath: Path.join(__dirname,'public'), staticPrefix: '/venue/', ipFilter: CloudFlareIpFilter @@ -126,7 +128,7 @@ async function main(){ debug('started') console.log('partying') - +/* await loadService(runnerRouter, { enabled: true, domain: 'postquantum.one', @@ -135,7 +137,7 @@ async function main(){ sendFullErrors: true, useNative: false - }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json') + }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json')*/ } From 015a46ba8cf4d22d4ca8e046bc5941d3afd94058 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 25 May 2026 17:17:31 +0000 Subject: [PATCH 06/50] debounce start --- src/config/json-file.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config/json-file.js b/src/config/json-file.js index f2f1bec..d52eb4f 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -23,6 +23,7 @@ class JsonFileConfig extends IConfig { this.defaults = defaults || {} this.content = Object.assign({}, this.defaults) this.writing = false + this.started = false } async load(){ @@ -48,11 +49,16 @@ class JsonFileConfig extends IConfig { } async start () { + + if(this.started){return} + await this.touchDir('') await this.load() fs.watchFile(this.path, this.handleFileChange.bind(this)) logger('started') + + this.started = true } async clear () { From 1bd2b5c4367bdca1190b000d2e574c67b6211d8a Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 29 May 2026 08:32:28 +0000 Subject: [PATCH 07/50] build with tar of static files and add file hashes & signing to service builds --- package.json | 2 + src/service/iservice.js | 12 +- src/service/service-builder.js | 79 +++++++- src/venue/build.js | 114 ++++++++++- src/venue/endpoints/create-package.js | 189 ++++++++++++++++++ .../{create-service.js => file-create.js} | 16 +- src/venue/endpoints/file-download.js | 0 src/venue/endpoints/file-write.js | 0 ...hmeral-session.js => ephemeral-session.js} | 2 +- src/venue/schema/venue_package.js | 1 + src/venue/schema/venue_service.js | 6 - src/venue/venue-host.js | 2 +- src/venue/venue-service.js | 10 +- 13 files changed, 407 insertions(+), 26 deletions(-) create mode 100644 src/venue/endpoints/create-package.js rename src/venue/endpoints/{create-service.js => file-create.js} (89%) create mode 100644 src/venue/endpoints/file-download.js create mode 100644 src/venue/endpoints/file-write.js rename src/venue/middleware/pre/{ephmeral-session.js => ephemeral-session.js} (97%) diff --git a/package.json b/package.json index 4687a73..cf9c05e 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", "git-repo-info": "^2.1.1", + "glob": "^13.0.6", "joi": "^17.13.3", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", @@ -108,6 +109,7 @@ "simple-peer": "9.11.1", "source-map": "^0.7.3", "store-js": "^2.0.4", + "tar": "^7.5.15", "tingodb": "^0.6.1", "touch": "^3.1.0", "url-parse": "^1.4.7", diff --git a/src/service/iservice.js b/src/service/iservice.js index b9f6c01..f8a8b8b 100644 --- a/src/service/iservice.js +++ b/src/service/iservice.js @@ -17,7 +17,7 @@ module.exports = class IService { * @param {*} build */ constructor({ - name, version, githash='', branch='' + name, version, githash='', branch='', owner=null }, build){ this.constructors = { @@ -48,11 +48,13 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: null + auth: null, + files: [], + files_root: null } this.compiled = { - package: { name, version, githash, branch }, + package: {owner, name, version, githash, branch }, schemas: { IndexSettings: {}, JSONSchema: [], @@ -70,7 +72,9 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: {} + auth: {}, + files: {}, + signatures: [] } this.compileSettings = { diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 614afd9..0f449fd 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -8,6 +8,15 @@ const BouncerDb = require('@dataparty/bouncer-db') const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') +const dataparty_crypto = require('@dataparty/crypto') + +const { + globSync +} = require('glob') +const { isArray } = require('lodash') + +const tar = require('tar') + //const IService = require('../iservice') @@ -27,7 +36,7 @@ module.exports = class ServiceBuilder { * @param {boolean} writeFile When true, files will be written. Defaults to `true` * @returns */ - async compile(outputPath, writeFile=true){ + async compile(outputPath, writeFile=true, owner){ if(!outputPath){ throw new Error('no output path') @@ -48,18 +57,34 @@ module.exports = class ServiceBuilder { this.compileList('tasks'), this.compileList('topics'), this.compileFile('auth'), - this.compileSchemas() + this.compileSchemas(), + this.compressFiles(outputPath, writeFile) ]) + + this.service.compiled.middleware_order = this.service.middleware_order this.service.compiled.compileSettings = this.service.compileSettings + if(owner){ + this.service.compiled.package.owner = owner.key.hash + const ownerSig = await owner.sign(this.service.compiled, true) + + console.log('sign keys', Object.keys(this.service.compiled)) + console.log(this.service.compiled.signature) + + this.service.compiled.signatures = { + [owner.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + } + + if(writeFile){ - const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-service.json' + const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.service.venue.json' fs.writeFileSync(buildOutput, JSON.stringify(this.service.compiled, null,2)) - const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-schema.json' + const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.schema.venue.json' fs.writeFileSync(schemaOutput, JSON.stringify({ package: this.service.compiled.package, ...this.service.compiled.schemas @@ -315,4 +340,50 @@ module.exports = class ServiceBuilder { this.service.sources.auth = auth_path this.service.constructors.auth = TopicClass } + + addFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.service.files){ + this.service.sources.files = result + } else { + this.service.sources.files = this.service.sources.files.concat(result) + } + + this.service.sources.files_root = root + + debug('addFiles',result) + + } + + async compressFiles(outputPath, writeFile){ + + let fileMap={} + + let files = this.service.sources.files.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + this.service.compiled.files = fileMap + + await tar.create({ + cwd: this.service.sources.files_root, + gzip: true, + file: Path.join(outputPath, this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz') + }, this.service.sources.files) + + } } \ No newline at end of file diff --git a/src/venue/build.js b/src/venue/build.js index 756f098..6dcea3a 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -4,9 +4,119 @@ const debug = require('debug')('build') const Dataparty = require('../index.js') +const dataparty_crypto = require('@dataparty/crypto') + const Pkg = require('../../package.json') const VenueService = require('./venue-service') +/** + * build/ + * - service.venue.json + * - schema.venue.json + * - client.venue.json + * - staticFiles.venue.tgz + * - package.venue.json + * { author, venue, files[], signatures{} } + */ + + +/* + +venue: { + url, + publicKey, + +} + +.venue-admin/ + - config/config.json + - db/ + - packages/ + - AUTHOR/PACKAGE-NAME + - package.venue.json + - service.venue.json + - static-files.venue.tgz + - parties/ + - PARTY_PUBLIC/ + - config/config.json (optional) + - db/ (optional) + - static-files.venue.tgz + - projects/ + - AUTHOR/PROJECT_PUBLIC + - project.venue.json + - static-files.venue.tgz + - party/ + - PARTY_NAME/ + - static-files.venue.tgz + + +~/code/my_venue_project/ + - package.json + - build.js + - public/ + - party/NAME + - default-config.json + - public/ + - package/NAME + - venue.json + { + projects: { + NAME: { + name: string + domain, (optional) + venue, (optional) + parties: [NAME], + routes: { + PREFIX: [ { PACKAGE(owner,name,version), party } ] + } + } + }, + packages: { + NAME: { + name: String + version: String, (optional) + service: localPathToService.js OR built service.json + } + } + } + +*/ + +async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, existingBuildPath, staticFilePaths}){ + + /* + + 0. create service.venue.json + 1. create schema.venue.json + 2. create client.venue.json + 3. create staticFiles.venue.tgz + 4. create package.venue.json + + 6. upload to venue + - create-package + * service.venue.json + * package.venue.json + + 7. upload files to venue + - create-files + * staticFiles.venue.tgz + * schema.venue.json + * client.venue.json + + 8. + + */ + +} + + +/** + * + * 1. generate admin/developer key + * 2. add venue identity (by url) + * 3. venue package build - build a package at some path + * 4. venue package push - c + */ async function main(){ const service = new VenueService({ @@ -14,8 +124,10 @@ async function main(){ version: Pkg.version }) + let a = await dataparty_crypto.Identity.fromRandomSeed(); + const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(Path.join(__dirname,'./dataparty'), true) + const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, a) debug('compiled') } diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js new file mode 100644 index 0000000..3e63cfa --- /dev/null +++ b/src/venue/endpoints/create-package.js @@ -0,0 +1,189 @@ +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Message, Routines} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-package') + +const IEndpoint = require('../../service/iendpoint') + +module.exports = class CreatePkgEndpoint extends IEndpoint { + + static get Name(){ + return 'create-package' + } + + + static get Description(){ + return 'Create venue package' + } + + { + venue_package: { + package:{ + owner: String, + info: { + name, version, githash, branch + }, + files: [ //package, service, static.tgz, + {hash: String, name: String, size: Number, signature} + ], + statics: { + PREFIX: [localPathGlob] + } + }, + trust: { + owner: signature + } + + } + } + + { + venue_project: { + + project:{ + + owner: String, + domain: String, + venue: String, + + parties: { + NAME: { + keys: {public, private: Secret(private)}, + defaultConfig: Object, + files: [ //static.tgz, + {hash: String, name: String, signature} + ], + statics: {prefix, [localPath]} + } + }, + + + + + routes: { + PREFIX: [{ + party: String, + package: { + owner: String, + info: {name, version, branch, githash}, + settings: { + sendFullErrors, + useNative + } + } + }] + } + }, + trust: { + owner: signature + } + } + } + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + validate: Joi.object().keys({ + /*settings: Joi.object().keys({ + enabled: Joi.boolean().default(true).required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), + sendFullErrors: Joi.boolean().default(false).required(), + useNative: Joi.boolean().default(false).required() + }).required(),*/ + service: Joi.object().keys({ + package: Joi.object().keys({ + name: Joi.string().required(), + version: Joi.string().required(), + githash: Joi.string().required(), + branch: Joi.string().required() + }).required(), + schemas: Joi.object().keys(null), + documents: Joi.object().keys(null), + endpoints: Joi.object().keys(null), + middleware: Joi.object().keys(null), + middleware_order: Joi.object().keys(null), + tasks: Joi.object().keys(null), + topics: Joi.object().keys(null), + auth: Joi.object().keys(null), + compileSettings: Joi.object().keys(null) + }).required(), + signature: Joi.object().keys({ + timestamp: Joi.number().required(), + type: Joi.string().required(), + value: Joi.string().required() + }).required() + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + //verify sender is admin + + ctx.debug('hello') + debug('echo') + ctx.debug('ctx.input', ctx.input) + + const compiledSrv = JSON.parse(ctx.input.service) + const serviceId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!srvDoc){ + debug('creating service') + srvDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(srvDoc.data) + await srvDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(srvDoc.data) + + debug('saving doc') + + + await srvDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + } + + return {srv:srvDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/endpoints/create-service.js b/src/venue/endpoints/file-create.js similarity index 89% rename from src/venue/endpoints/create-service.js rename to src/venue/endpoints/file-create.js index ac3e324..d2e20d6 100644 --- a/src/venue/endpoints/create-service.js +++ b/src/venue/endpoints/file-create.js @@ -1,19 +1,19 @@ const Joi = require('joi') const Hoek = require('@hapi/hoek') const {Message, Routines} = require('@dataparty/crypto') -const debug = require('debug')('dataparty.endpoint.create-service') +const debug = require('debug')('dataparty.endpoint.file-create') const IEndpoint = require('../../service/iendpoint') -module.exports = class CreateSrvEndpoint extends IEndpoint { +module.exports = class FileCreateEndpoint extends IEndpoint { static get Name(){ - return 'create-service' + return 'file-create' } static get Description(){ - return 'Create venue service' + return 'Create venue package file' } static get MiddlewareConfig(){ @@ -21,13 +21,13 @@ module.exports = class CreateSrvEndpoint extends IEndpoint { pre: { decrypt: true, validate: Joi.object().keys({ - settings: Joi.object().keys({ + /*settings: Joi.object().keys({ enabled: Joi.boolean().default(true).required(), - domain: Joi.string().required(), - prefix: Joi.string().default('').required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), sendFullErrors: Joi.boolean().default(false).required(), useNative: Joi.boolean().default(false).required() - }).required(), + }).required(),*/ service: Joi.object().keys({ package: Joi.object().keys({ name: Joi.string().required(), diff --git a/src/venue/endpoints/file-download.js b/src/venue/endpoints/file-download.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/endpoints/file-write.js b/src/venue/endpoints/file-write.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/middleware/pre/ephmeral-session.js b/src/venue/middleware/pre/ephemeral-session.js similarity index 97% rename from src/venue/middleware/pre/ephmeral-session.js rename to src/venue/middleware/pre/ephemeral-session.js index 11f862e..de0fb2a 100644 --- a/src/venue/middleware/pre/ephmeral-session.js +++ b/src/venue/middleware/pre/ephemeral-session.js @@ -3,7 +3,7 @@ const Hoek = require('@hapi/hoek') const {Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') -const IMiddleware = require('../../service/imiddleware') +const IMiddleware = require('../../../service/imiddleware') module.exports = class Decrypt extends IMiddleware { diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index 261f55e..a56929e 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -19,6 +19,7 @@ class VenuePkg extends ISchema { venue: {type: String}, settings: { enabled: {type: Boolean, required: true}, + staticPrefix: String, sendFullErrors: {type: Boolean, required: true}, useNative: {type: Boolean, required: true}, defaultConfig: {type: Object} diff --git a/src/venue/schema/venue_service.js b/src/venue/schema/venue_service.js index f66ff9e..181a556 100644 --- a/src/venue/schema/venue_service.js +++ b/src/venue/schema/venue_service.js @@ -30,12 +30,6 @@ class VenueSrv extends ISchema { version: {type: String, required: true}, githash: {type: String, required: true}, branch: {type: String, required: true} - }, - compressedBuild: {type: String, required: true}, //! zlib compressed - signature: { - timestamp: {type: Number, required: true}, - type: {type: String, required: true, maxlength: 10}, - value: {type: String, required: true} } } } diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index b3f8ab2..c16f413 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -103,7 +103,7 @@ async function main(){ party, service, sendFullErrors: true, useNative: false, - prefix: 'api/' + prefix: 'venue/' }) let runnerRouter = new Dataparty.RunnerRouter(runner) diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index dc56ea1..d5eb35e 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -19,6 +19,8 @@ class VenueService extends DatapartySrv.IService { builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) + builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) + builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) @@ -27,12 +29,18 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) - builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) + //builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) builder.addAuth(Path.join(__dirname, './auth.js')) + + builder.addFiles(__dirname, [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ], { nodir: true, follow: true }) } } From aac16e10b886fadbd288efc14545fe486c6b9ee1 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 30 May 2026 15:37:53 +0000 Subject: [PATCH 08/50] packages uploading during build but not finished yet --- examples/secure-config-argon2.js | 1 - package.json | 2 +- src/comms/peer-comms.js | 13 ++++++- src/party/peer/match-maker-client.js | 7 ++-- src/service/endpoints/service-version.js | 3 +- src/service/middleware/post/encrypt.js | 6 ++- src/service/middleware/pre/decrypt.js | 10 ++++- src/service/service-host.js | 4 +- src/venue/build.js | 49 +++++++++++++++++++++++- src/venue/endpoints/create-package.js | 43 ++++++++++++++++----- src/venue/endpoints/key-announce.js | 7 +--- src/venue/middleware/pre/decrypt-nacl.js | 2 +- src/venue/venue-host.js | 4 +- src/venue/venue-service.js | 2 +- 14 files changed, 120 insertions(+), 33 deletions(-) diff --git a/examples/secure-config-argon2.js b/examples/secure-config-argon2.js index d4df83f..ce419d4 100644 --- a/examples/secure-config-argon2.js +++ b/examples/secure-config-argon2.js @@ -18,7 +18,6 @@ const HELP_INFO = ` ` async function main(){ - const memoryConfig = new Dataparty.Config.MemoryConfig({foo: 'bar'}) const jsonConfig = new Dataparty.Config.JsonFileConfig({ diff --git a/package.json b/package.json index cf9c05e..7c295b1 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "express-list-routes": "^1.1.9", "git-repo-info": "^2.1.1", "glob": "^13.0.6", - "joi": "^17.13.3", + "joi": "^18.2.1", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", "jsonpath-plus": "^0.20.1", diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index b64c98a..b20fd16 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -458,10 +458,18 @@ class PeerComms extends ISocketComms { return } + debug('input type', typeof op.input.data, Object.keys(op.input.data)) + debug('op.msg type', typeof op.msg, Object.keys(op.msg), Buffer.isBuffer(op.msg)) + + let bodyValue = Buffer.isBuffer(op.msg) ? + op.input.data : + //Routines.BSON.parseObject(new Routines.BSON.BaseParser( op.msg )) : + JSON.parse(op.msg.toString()) + const req = HttpMocks.createRequest({ method: 'GET', url: '/'+op.input.endpoint, - body: (op.input.data) ? JSON.parse(op.msg.toString()) : undefined + body: bodyValue }) const res = HttpMocks.createResponse() @@ -474,6 +482,9 @@ class PeerComms extends ISocketComms { debug('route',route) + req.peer = this + req.source = 'PeerComms' + debug('call route', await route._events.route({ method: req.method, pathname: req.url, diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index cf2f6bd..7c9b9ff 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -2,20 +2,19 @@ const EventEmitter = require('eventemitter3') const debug = require('debug')('dataparty.match-maker-client') - const dataparty_crypto = require('@dataparty/crypto') const LokiParty = require('../local/loki-party') const PeerParty = require('./peer-party') const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws', billingIdentity=null){ + constructor(identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws', billingIdentity=null){ super() - this.contacts = contacts this.sessionKey = null @@ -72,7 +71,7 @@ class MatchMakerClient extends EventEmitter { await this.restParty.start() if(!this.restParty.comms){ - this.restParty.comms = new Dataparty.Comms.RestComms({ + this.restParty.comms = new RestComms({ party:this.restParty, config: this.restParty.config }) diff --git a/src/service/endpoints/service-version.js b/src/service/endpoints/service-version.js index b84c91e..ab5ed11 100644 --- a/src/service/endpoints/service-version.js +++ b/src/service/endpoints/service-version.js @@ -27,7 +27,8 @@ module.exports = class ServiceVersion extends IEndpoint { name: Joi.string(), branch: Joi.string(), version: Joi.string(), - githash: Joi.string() + githash: Joi.string(), + owner: Joi.string(), }) } } diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index a0912d8..8645087 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -30,7 +30,11 @@ module.exports = class Encrypt extends IMiddleware { static async run(ctx, {Config}){ if (!Config){ return } - + + if(ctx.req.source && ctx.req.source == 'PeerComms'){ + ctx.setOutput(ctx.output) + return + } const senderStr = JSON.stringify({key: ctx.senderKey}) diff --git a/src/service/middleware/pre/decrypt.js b/src/service/middleware/pre/decrypt.js index 20adf0c..048dd52 100644 --- a/src/service/middleware/pre/decrypt.js +++ b/src/service/middleware/pre/decrypt.js @@ -31,8 +31,16 @@ module.exports = class Decrypt extends IMiddleware { if (!Config){ return } + + if(!context.input || !context.input.enc){ - throw new Error('insecure message') + + if(!context.req.source || context.req.source != 'PeerComms'){ + throw new Error('insecure message -' + context.req.source) + } + + context.setInput( context.input ) + return } context.debug('input', context.input, typeof context.input) diff --git a/src/service/service-host.js b/src/service/service-host.js index a03a5dd..10033fa 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -113,8 +113,8 @@ class ServiceHost { if(debug.enabled){ this.apiApp.use(morgan('combined')) } this.apiApp.use(bodyParser.urlencoded({ extended: true })) - this.apiApp.use(bodyParser.json()) - this.apiApp.use(bodyParser.raw()) + this.apiApp.use(bodyParser.json({limit:'10MB'})) + this.apiApp.use(bodyParser.raw({limit:'10MB'})) this.apiApp.set('trust proxy', trust_proxy) diff --git a/src/venue/build.js b/src/venue/build.js index 6dcea3a..4ba65fc 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -1,3 +1,4 @@ +const fs = require('fs') const Path = require('path') const debug = require('debug')('build') @@ -118,18 +119,62 @@ async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, exi * 4. venue package push - c */ +async function pushService(devId, build){ + + let client = new Dataparty.MatchMakerClient( + devId, + null, + 'https://api.dataparty.xyz/venue', + 'wss://api.dataparty.xyz/ws' + ) + + await client.start() + + const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') + + console.log('is staticTar a buffer? ', staticTar instanceof Buffer); // true + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar: staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: false + }) + + console.log('result', uploadResult) +} + async function main(){ const service = new VenueService({ name: '@dataparty/venue', version: Pkg.version }) - let a = await dataparty_crypto.Identity.fromRandomSeed(); + const path = Path.join(process.env.HOME, '.venue-admin') + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + let party = new Dataparty.TingoParty({ + path: path+'/db', + config, + noCache: false + }) + + await party.start() + + console.log( 'identity - ', party.identity.key.hash ) + const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, a) + const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, party.privateIdentity) debug('compiled') + + await pushService( party.privateIdentity, build ) + + + + } main().catch(err=>{ diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 3e63cfa..517257c 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -16,6 +16,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { return 'Create venue package' } + /* { venue_package: { package:{ @@ -80,10 +81,14 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { } } + */ + static get MiddlewareConfig(){ return { pre: { decrypt: true, + ephemeral_session: true, + //validate: Joi.object().keys(null) validate: Joi.object().keys({ /*settings: Joi.object().keys({ enabled: Joi.boolean().default(true).required(), @@ -92,8 +97,9 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { sendFullErrors: Joi.boolean().default(false).required(), useNative: Joi.boolean().default(false).required() }).required(),*/ - service: Joi.object().keys({ + build: Joi.object().keys({ package: Joi.object().keys({ + owner: Joi.string(), name: Joi.string().required(), version: Joi.string().required(), githash: Joi.string().required(), @@ -107,13 +113,11 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { tasks: Joi.object().keys(null), topics: Joi.object().keys(null), auth: Joi.object().keys(null), + files: Joi.object().keys(null), + signatures: Joi.object().keys(null).required(), compileSettings: Joi.object().keys(null) }).required(), - signature: Joi.object().keys({ - timestamp: Joi.number().required(), - type: Joi.string().required(), - value: Joi.string().required() - }).required() + staticTar: Joi.binary() }) }, post: { @@ -125,12 +129,31 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { static async run(ctx){ - //verify sender is admin + ctx.debug('hello') debug('echo') ctx.debug('ctx.input', ctx.input) + + //verify sender is admin + const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) + if(!isAdmin){ + ctx.debug('non-admin user') + return {done: false} + } + + // verify build signature + + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + const compiledSrv = JSON.parse(ctx.input.service) const serviceId = compiledSrv.package.name + '-' + compiledSrv.package.version debug('addService', serviceId) @@ -182,8 +205,10 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { console.log(err) } debug('updated service') - } + }*/ + + return {done: true} - return {srv:srvDoc.data} + //return {srv:srvDoc.data} } } \ No newline at end of file diff --git a/src/venue/endpoints/key-announce.js b/src/venue/endpoints/key-announce.js index de1e7ec..6b281b6 100644 --- a/src/venue/endpoints/key-announce.js +++ b/src/venue/endpoints/key-announce.js @@ -123,16 +123,11 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { await actorSigMsg.assertVerified( actorIdentity, true ) await sessionSigMsg.assertVerified( sessionIdentity, true ) - // verify key-hash is an admin - //const admins = (await ctx.party.config.read('admins')) || [] - - //if(admins.indexOf(computedActorHash) == -1){ + // verify actor is an admin const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) if(!isAdmin){ ctx.debug('non-admin user') return {done: false} - - process.exit(1) } diff --git a/src/venue/middleware/pre/decrypt-nacl.js b/src/venue/middleware/pre/decrypt-nacl.js index 069812a..857f2e9 100644 --- a/src/venue/middleware/pre/decrypt-nacl.js +++ b/src/venue/middleware/pre/decrypt-nacl.js @@ -32,7 +32,7 @@ module.exports = class DecryptNaCl extends IMiddleware { if (!Config){ return } if(!context.input || !context.input.enc){ - throw new Error('insecure message') + throw new Error('insecure message - here') } context.debug('input', context.input, typeof context.input) diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index c16f413..7118e18 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -5,8 +5,8 @@ const Dataparty = require('../index') const VenueService = require('./venue-service') -const VenueServiceSchema = require('./dataparty/@dataparty-venue.dataparty-schema.json') -const VenueSrv = require('./dataparty/@dataparty-venue.dataparty-service.json') +const VenueServiceSchema = require('./dataparty/@dataparty-venue.schema.venue.json') +const VenueSrv = require('./dataparty/@dataparty-venue.service.venue.json') async function loadService(runnerRouter, settings, serviceFilePath){ diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index d5eb35e..d100d12 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -29,7 +29,7 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) - //builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) From c022c128e0e64349b2c16361c27b92f2c71a3988 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 30 May 2026 21:09:28 +0000 Subject: [PATCH 09/50] hash tar file during build --- src/service/service-builder.js | 18 ++++++++++++++++-- src/venue/build.js | 10 +++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 0f449fd..0137f93 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -377,13 +377,27 @@ module.exports = class ServiceBuilder { return hash }) - this.service.compiled.files = fileMap + const tarFileName = this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) await tar.create({ cwd: this.service.sources.files_root, gzip: true, - file: Path.join(outputPath, this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz') + file: tarPath }, this.service.sources.files) + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + this.service.compiled.files = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } } } \ No newline at end of file diff --git a/src/venue/build.js b/src/venue/build.js index 4ba65fc..7f303b8 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -119,7 +119,7 @@ async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, exi * 4. venue package push - c */ -async function pushService(devId, build){ +async function pushService(devId, build, staticTar){ let client = new Dataparty.MatchMakerClient( devId, @@ -130,9 +130,6 @@ async function pushService(devId, build){ await client.start() - const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') - - console.log('is staticTar a buffer? ', staticTar instanceof Buffer); // true let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar: staticTar}, { expectClearTextReply: false, @@ -170,7 +167,10 @@ async function main(){ debug('compiled') - await pushService( party.privateIdentity, build ) + const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') + + debug('is staticTar a buffer? ', staticTar instanceof Buffer); // true + //await pushService( party.privateIdentity, build, staticTar ) From 09ab900a4d2187848c80cf5b35f80bc453930a70 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 31 May 2026 15:19:41 +0000 Subject: [PATCH 10/50] verifying service & tars, saving files to disc --- src/config/json-file.js | 10 ++ src/service/iservice.js | 2 +- src/venue/build.js | 6 +- src/venue/endpoints/create-package.js | 126 ++++++++++++++++++++++---- src/venue/venue-host.js | 2 +- 5 files changed, 125 insertions(+), 21 deletions(-) diff --git a/src/config/json-file.js b/src/config/json-file.js index d52eb4f..a8376c8 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -110,6 +110,16 @@ class JsonFileConfig extends IConfig { }) } + fileExists(path){ + var realPath = Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + + return fs.existsSync(realPath) + } + + filePath(path){ + return Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + } + async handleFileChange(current, previous){ if(this.writing){ return } diff --git a/src/service/iservice.js b/src/service/iservice.js index f8a8b8b..2d46caf 100644 --- a/src/service/iservice.js +++ b/src/service/iservice.js @@ -74,7 +74,7 @@ module.exports = class IService { topics: {}, auth: {}, files: {}, - signatures: [] + //signatures: {} } this.compileSettings = { diff --git a/src/venue/build.js b/src/venue/build.js index 7f303b8..e126d57 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -131,10 +131,10 @@ async function pushService(devId, build, staticTar){ await client.start() - let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar: staticTar}, { + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { expectClearTextReply: false, sendClearTextRequest: false, - useSessions: false + useSessions: true }) console.log('result', uploadResult) @@ -170,7 +170,7 @@ async function main(){ const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') debug('is staticTar a buffer? ', staticTar instanceof Buffer); // true - //await pushService( party.privateIdentity, build, staticTar ) + await pushService( party.privateIdentity, build, staticTar ) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 517257c..9a4022d 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -1,10 +1,21 @@ +const fs = require('fs') const Joi = require('joi') const Hoek = require('@hapi/hoek') -const {Message, Routines} = require('@dataparty/crypto') +const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-package') + const IEndpoint = require('../../service/iendpoint') +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + module.exports = class CreatePkgEndpoint extends IEndpoint { static get Name(){ @@ -88,15 +99,14 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { pre: { decrypt: true, ephemeral_session: true, - //validate: Joi.object().keys(null) validate: Joi.object().keys({ - /*settings: Joi.object().keys({ - enabled: Joi.boolean().default(true).required(), + settings: Joi.object().keys({ + //enabled: Joi.boolean().default(true).required(), //domain: Joi.string().required(), staticPrefix: Joi.string().default('/'), - sendFullErrors: Joi.boolean().default(false).required(), - useNative: Joi.boolean().default(false).required() - }).required(),*/ + sendFullErrors: Joi.boolean().default(false), + useNative: Joi.boolean().default(false) + }), build: Joi.object().keys({ package: Joi.object().keys({ owner: Joi.string(), @@ -117,7 +127,8 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { signatures: Joi.object().keys(null).required(), compileSettings: Joi.object().keys(null) }).required(), - staticTar: Joi.binary() + //staticTar: Joi.binary() + staticTar: Joi.any().custom(typedArraySchema) }) }, post: { @@ -129,22 +140,105 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { static async run(ctx){ + let {signatures, ...buildWithoutSig} = ctx.input.build + + const pkgOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.build.package.owner ) + .exec() + )[0] + + if(!pkgOwnerIdentityDoc){ + throw new Error('package owner not authorized') + } + + debug('found pkg owner', pkgOwnerIdentityDoc.hash) + + const pkgOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: pkgOwnerIdentityDoc.data.type, + hash: pkgOwnerIdentityDoc.data.hash, + public: pkgOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + //debug('build', buildWithoutSig) + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.build.package.owner]) + + let signedBuildMsg = new Message({ + msg: buildWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') - ctx.debug('hello') - debug('echo') - ctx.debug('ctx.input', ctx.input) + await signedBuildMsg.assertVerified(pkgOwnerIdentity, true) + debug('verified package signature') - //verify sender is admin - const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) - if(!isAdmin){ - ctx.debug('non-admin user') - return {done: false} + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const safeFileName = ctx.input.build.package.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' + + const buildFiles = ctx.input.build.files[tarFileName] + + if(buildFiles && buildFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match package definition") } + debug('verified staticTar') + + debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) + + const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(buildWithoutSig) + + const buildHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + buildBSON + ) + ) + + const safeBuildHash = buildHash.replace('/', '-') + + debug('\t'+'hash', buildHash) + + const buildWorkspace = 'packages/'+safeFileName+'/'+ctx.input.build.package.version+'/'+safeBuildHash + + const config = ctx.party.config + + const workspacePath = await config.touchDir(buildWorkspace) + + + debug('\t'+'workspace - local', buildWorkspace) + debug('\t'+'workspace - global', workspacePath) + + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) + // verify build signature + //ctx.input. // untar listed files diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index 7118e18..6e72731 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -88,7 +88,7 @@ async function main(){ let party = new Dataparty.TingoParty({ path: path+'/db', model: VenueServiceSchema, - config: new Dataparty.Config.JsonFileConfig({basePath: path+'/config'}), + config: new Dataparty.Config.JsonFileConfig({basePath: path}), noCache: false }) From 1c794adcaf38951a5902accf812b59462d178c73 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 31 May 2026 17:18:26 +0000 Subject: [PATCH 11/50] services written to db --- src/service/service-builder.js | 11 +++++++ src/venue/endpoints/create-package.js | 42 +++++++++++++++++++++++++-- src/venue/schema/venue_package.js | 15 ++++------ src/venue/venue-service.js | 2 +- 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 0137f93..08b8f94 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -7,6 +7,7 @@ const gitRepoInfo = require('git-repo-info') const BouncerDb = require('@dataparty/bouncer-db') const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') +const zlib = require('zlib') const dataparty_crypto = require('@dataparty/crypto') @@ -89,6 +90,16 @@ module.exports = class ServiceBuilder { package: this.service.compiled.package, ...this.service.compiled.schemas }, null, 2)) + + // Gzip compression (most common for HTTP) + const compressed = zlib.gzipSync(JSON.stringify(this.service.compiled, null,2)); + + // Brotli compression (better ratio, Node.js 10.5.0+) + const compressedBrotli = zlib.brotliCompressSync(JSON.stringify(this.service.compiled, null,2)); + + console.log('Original:', JSON.stringify(this.service.compiled, null,2).length, 'bytes'); + console.log('Gzip:', compressed.length, 'bytes'); + console.log('Brotli:', compressedBrotli.length, 'bytes'); } return this.service.compiled diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 9a4022d..fbe7a28 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -3,7 +3,7 @@ const Joi = require('joi') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-package') - +const zlib = require('zlib') const IEndpoint = require('../../service/iendpoint') @@ -105,7 +105,8 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { //domain: Joi.string().required(), staticPrefix: Joi.string().default('/'), sendFullErrors: Joi.boolean().default(false), - useNative: Joi.boolean().default(false) + useNative: Joi.boolean().default(false), + defaultConfig: Joi.object().keys(null) }), build: Joi.object().keys({ package: Joi.object().keys({ @@ -199,7 +200,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) - const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(buildWithoutSig) + const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(/*ctx.input.build*/buildWithoutSig) const buildHash = Routines.Utils.base64.encode( Routines.Utils.hash( @@ -235,6 +236,41 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { Path.join(workspacePath, safeFileName+'.service.venue.json'), JSON.stringify(ctx.input.build, null, 2) ) + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.build)) + + const build = ctx.input.build + const serviceId = build.package.name + '@' + build.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_pkg') + .where('package.name').equals(build.package.name) + .where('package.version').equals(build.package.version) + .where('hash').equals(buildHash) + .exec())[0] + + + if(!srvDoc){ + debug('creating service') + + const {owner, ...pkgWithoutOwner} = build.package + + srvDoc = await ctx.party.createDocument('venue_pkg', { + owner: build.package.owner, + 'created': Date.now(), + venue: ctx.party.identity.key.hash, + hash: buildHash, + workspace: workspacePath, + settings: ctx.input.settings, + package: pkgWithoutOwner, + compressedBuild: Routines.Utils.base64.encode(compressedBrotliBuild) + }) + + debug('service created') + } else { + debug('need to update service') + } // verify build signature diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index a56929e..a205839 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -17,8 +17,10 @@ class VenuePkg extends ISchema { created: {type: Number, required: true}, changed: {type: Number}, venue: {type: String}, + workspace: {type: String, required: true}, + hash: {type: String, required: true, index: true}, settings: { - enabled: {type: Boolean, required: true}, + //enabled: {type: Boolean, required: true}, staticPrefix: String, sendFullErrors: {type: Boolean, required: true}, useNative: {type: Boolean, required: true}, @@ -26,16 +28,11 @@ class VenuePkg extends ISchema { }, package: { name: {type: String, required: true, index: true}, - version: {type: String, required: true}, + version: {type: String, required: true, index: true}, githash: {type: String, required: true}, - branch: {type: String, required: true} + branch: {type: String, required: true}, }, - compressedBuild: {type: String, required: true}, //! zlib compressed - signature: { - timestamp: {type: Number, required: true}, - type: {type: String, required: true, maxlength: 10}, - value: {type: String, required: true} - } + compressedBuild: {type: String, required: true} //! brotli compressed } } diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index d100d12..c0f0a30 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -14,7 +14,7 @@ class VenueService extends DatapartySrv.IService { builder.addSchema(Path.join(__dirname, './schema/public-key.js')) builder.addSchema(Path.join(__dirname, './schema/session-key.js')) - builder.addSchema(Path.join(__dirname, './schema/venue_service.js')) + builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) From 2b68a1d77d22f11483b3712de8e40717377af5df Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 31 May 2026 17:32:01 +0000 Subject: [PATCH 12/50] regular expression for safeworkspace path and move file save to after db is updated --- src/venue/endpoints/create-package.js | 30 ++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index fbe7a28..30ff2bd 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -208,7 +208,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { ) ) - const safeBuildHash = buildHash.replace('/', '-') + const safeBuildHash = buildHash.replace(/\//g, "-") debug('\t'+'hash', buildHash) @@ -222,20 +222,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('\t'+'workspace - local', buildWorkspace) debug('\t'+'workspace - global', workspacePath) - fs.writeFileSync( - Path.join(workspacePath, tarFileName), - ctx.input.staticTar - ) - - /*fs.writeFileSync( - Path.join(workspacePath, safeFileName+'.service.venue.bson'), - Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) - )*/ - fs.writeFileSync( - Path.join(workspacePath, safeFileName+'.service.venue.json'), - JSON.stringify(ctx.input.build, null, 2) - ) const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.build)) @@ -271,6 +258,21 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { } else { debug('need to update service') } + + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) // verify build signature From 189090796e51d762bdf98f4dab8b09ec3173a2c5 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Tue, 2 Jun 2026 21:40:04 +0000 Subject: [PATCH 13/50] initial project schema --- src/venue/schema/venue-project.js | 82 +++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/venue/schema/venue-project.js diff --git a/src/venue/schema/venue-project.js b/src/venue/schema/venue-project.js new file mode 100644 index 0000000..51d77fa --- /dev/null +++ b/src/venue/schema/venue-project.js @@ -0,0 +1,82 @@ +'use strict' + +const debug = require('debug')('venue.venue_project') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenueProject extends ISchema { + + static get Type () { return 'venue_project' } + + static get Schema(){ + return { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + venue: {type: String}, + domain: {type: String, index: true, unique: true}, + + // i2p enable? + // standalone hosting? + + workspace: {type: String, required: true}, + //hash: {type: String, required: true, index: true}, + party: [{ + name: String, + type: {type: String, enum: ['TingoParty', 'LokiParty', 'PeerParty', 'MongoParty']}, + tingo: { + path: String + }, + loki: { + path: String + }, + peer: { + venue: String, + remoteIdentity: String + }, + key: { + hash: String, + securePrivate: String + }, + settings: { + noCache: Boolean, + }, + defaultConfig: Object + }], + routes: [{ + prefix: String, + party: String, + package: { + owner: String, + name: {type: String, required: true, index: true}, + version: String, + branch: String, + hash: String, + }, + settings: { + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + } + }] + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenueProject \ No newline at end of file From 7a08ea0663bc95bd3c4b6ad87f7e7f2772a5439c Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Tue, 9 Jun 2026 13:20:29 +0000 Subject: [PATCH 14/50] move reasable package parts --- .../endpoints/key-announce.js | 0 src/service/index.js | 22 +++++++++++++++---- .../middleware/pre/ephemeral-session.js | 2 +- src/{venue => service}/schema/public-key.js | 0 src/{venue => service}/schema/session-key.js | 0 src/venue/endpoints/create-package.js | 2 +- src/venue/venue-service.js | 13 +++++++---- 7 files changed, 29 insertions(+), 10 deletions(-) rename src/{venue => service}/endpoints/key-announce.js (100%) rename src/{venue => service}/middleware/pre/ephemeral-session.js (97%) rename src/{venue => service}/schema/public-key.js (100%) rename src/{venue => service}/schema/session-key.js (100%) diff --git a/src/venue/endpoints/key-announce.js b/src/service/endpoints/key-announce.js similarity index 100% rename from src/venue/endpoints/key-announce.js rename to src/service/endpoints/key-announce.js diff --git a/src/service/index.js b/src/service/index.js index bf2ed65..9f00a91 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -24,7 +24,8 @@ exports.MiddlewareRunner= require('./middleware-runner') exports.middleware = { pre: { decrypt: require('./middleware/pre/decrypt'), - validate: require('./middleware/pre/validate') + validate: require('./middleware/pre/validate'), + ephemeral_session: require('./middleware/pre/ephemeral-session.js') }, post: { validate: require('./middleware/post/validate.js'), @@ -35,7 +36,8 @@ exports.middleware = { exports.middleware_paths = { pre: { decrypt: Path.join(__dirname, './middleware/pre/decrypt.js'), - validate: Path.join(__dirname, './middleware/pre/validate.js') + validate: Path.join(__dirname, './middleware/pre/validate.js'), + ephemeral_session: Path.join(__dirname, './middleware/pre/ephemeral-session.js') }, post: { validate: Path.join(__dirname, './middleware/post/validate.js'), @@ -47,12 +49,24 @@ exports.endpoint = { echo: require('./endpoints/echo'), secureecho: require('./endpoints/secure-echo'), identity: require('./endpoints/service-identity'), - version: require('./endpoints/service-version') + version: require('./endpoints/service-version'), + key_announce: require('./endpoints/key-announce'), } exports.endpoint_paths = { echo: Path.join(__dirname, './endpoints/echo.js'), secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), - version: Path.join(__dirname, './endpoints/service-version.js') + version: Path.join(__dirname, './endpoints/service-version.js'), + key_announce: Path.join(__dirname, './endpoints/key-announce.js') +} + +exports.schema = { + public_key: require('./schema/public-key'), + session_key: require('./schema/session-key') +} + +exports.schema_paths = { + public_key: Path.join(__dirname, './schema/public-key.js'), + session_key: Path.join(__dirname, './schema/session-key.js') } \ No newline at end of file diff --git a/src/venue/middleware/pre/ephemeral-session.js b/src/service/middleware/pre/ephemeral-session.js similarity index 97% rename from src/venue/middleware/pre/ephemeral-session.js rename to src/service/middleware/pre/ephemeral-session.js index de0fb2a..fd099cf 100644 --- a/src/venue/middleware/pre/ephemeral-session.js +++ b/src/service/middleware/pre/ephemeral-session.js @@ -3,7 +3,7 @@ const Hoek = require('@hapi/hoek') const {Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') -const IMiddleware = require('../../../service/imiddleware') +const IMiddleware = require('../../imiddleware') module.exports = class Decrypt extends IMiddleware { diff --git a/src/venue/schema/public-key.js b/src/service/schema/public-key.js similarity index 100% rename from src/venue/schema/public-key.js rename to src/service/schema/public-key.js diff --git a/src/venue/schema/session-key.js b/src/service/schema/session-key.js similarity index 100% rename from src/venue/schema/session-key.js rename to src/service/schema/session-key.js diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 30ff2bd..4eedd2d 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -256,7 +256,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('service created') } else { - debug('need to update service') + debug('need to update service?') } fs.writeFileSync( diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index c0f0a30..a304e47 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -12,22 +12,27 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) - builder.addSchema(Path.join(__dirname, './schema/public-key.js')) - builder.addSchema(Path.join(__dirname, './schema/session-key.js')) + //builder.addSchema(Path.join(__dirname, './schema/public-key.js')) + //builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) + builder.addSchema(DatapartySrv.schema_paths.public_key) + builder.addSchema(DatapartySrv.schema_paths.session_key) + builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) + builder.addMiddleware(DatapartySrv.middleware_paths.pre.ephemeral_session) - builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) + //builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) builder.addEndpoint(DatapartySrv.endpoint_paths.identity) builder.addEndpoint(DatapartySrv.endpoint_paths.version) + builder.addEndpoint(DatapartySrv.endpoint_paths.key_announce) - builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) + //builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) From 237adf2b6764e5598bcdb0c3985b4780b94bff0a Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 10 Jun 2026 01:42:42 +0000 Subject: [PATCH 15/50] venued commands --- package.json | 14 +- src/bouncer/ischema.js | 2 +- src/service/endpoints/key-announce.js | 2 +- src/service/service-builder.js | 18 +- src/service/service-host.js | 10 +- src/venue/bin/commands/venued-admin-add.js | 85 ++++++ src/venue/bin/commands/venued-admin-list.js | 69 +++++ src/venue/bin/commands/venued-host.js | 277 ++++++++++++++++++++ src/venue/bin/venue.js | 71 +++++ src/venue/bin/venued.js | 82 ++++++ src/venue/schema/venue_package.js | 2 +- 11 files changed, 617 insertions(+), 15 deletions(-) create mode 100644 src/venue/bin/commands/venued-admin-add.js create mode 100644 src/venue/bin/commands/venued-admin-list.js create mode 100644 src/venue/bin/commands/venued-host.js create mode 100755 src/venue/bin/venue.js create mode 100755 src/venue/bin/venued.js diff --git a/package.json b/package.json index 7c295b1..7b16dcd 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,10 @@ "dist", "src/*" ], + "bin": { + "venue": "./src/venue/bin/venue.js", + "venued": "./src/venue/bin/venued.js" + }, "scripts": { "test": "npx lab", "build": "npx parcel build --no-scope-hoist", @@ -63,10 +67,10 @@ }, "dependencies": { "@babel/runtime": "^7.28.4", - "@dataparty/bouncer-db": "1.0.1", + "@dataparty/bouncer-db": "https://github.com/datapartyjs/bouncer-db#mongoose-latest", "@dataparty/crypto": "github:datapartyjs/dataparty-crypto", "@dataparty/tasker": "^0.0.3", - "@diva.exchange/i2p-sam": "^4.1.8", + "@diva.exchange/i2p-sam": "5.5.2", "@markwylde/liferaft": "^1.3.4", "@sevenbitbyte/ncc": "0.0.2", "ajv": "6.12.5", @@ -77,6 +81,7 @@ "buffer": "^6.0.3", "bufferutil": "^4.0.8", "colors": "1.3.1", + "command-tree": "github:datapartyjs/command-tree", "cors": "^2.8.5", "debug": "^3.1.0", "dom-storage": "^2.1.0", @@ -84,12 +89,13 @@ "express": "^4.17.1", "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", + "fast-safe-stringify": "^2.1.1", "git-repo-info": "^2.1.1", "glob": "^13.0.6", "joi": "^18.2.1", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", - "jsonpath-plus": "^0.20.1", + "jsonpath-plus": "10.4.0", "last-eventemitter": "^1.1.1", "lodash": "^4.17.21", "lokijs": "1.5.12", @@ -120,7 +126,7 @@ "zangodb": "https://github.com/sevenbitbyte/zangodb#hash-patch" }, "devDependencies": { - "@dataparty/bouncer-model": "1.4.3", + "@dataparty/bouncer-model": "https://github.com/datapartyjs/bouncer-model#mongoose-latest", "@hapi/code": "^9.0.1", "@hapi/joi": "^17.1.1", "@hapi/lab": "^25.0.1", diff --git a/src/bouncer/ischema.js b/src/bouncer/ischema.js index 78fd0a8..18c080b 100644 --- a/src/bouncer/ischema.js +++ b/src/bouncer/ischema.js @@ -1,4 +1,4 @@ -const debug = require('debug')('bouncer.ISchema') +const debug = require('debug')('dataparty.bouncer.ISchema') const MgoUtils = require('../utils/mongoose-scheme-utils') module.exports = class ISchema { diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index 6b281b6..b2768f5 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -5,7 +5,7 @@ const debug = require('debug')('dataparty.endpoint.key-announce') const {Identity, Message, Routines} = require('@dataparty/crypto') //const IEndpoint = require('@dataparty/api/src/service/iendpoint') -const IEndpoint = require('../../service/iendpoint') +const IEndpoint = require('../iendpoint') const KeyVerifier = Joi.object().keys({ id: Joi.string().max(100), diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 08b8f94..c78d3fe 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -9,6 +9,8 @@ const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') const zlib = require('zlib') +const safeStringify = require('fast-safe-stringify') + const dataparty_crypto = require('@dataparty/crypto') const { @@ -178,24 +180,32 @@ module.exports = class ServiceBuilder { this.service.compiled.schemas.Permissions[model.Type] = await model.permissions() this.service.compiled.schemas.JSONSchema.push(jsonSchema) + const safePaths = JSON.parse(safeStringify(schema.paths)) + + //debug(schema.paths) debug('\t','type',model.Type) let indexed = JSONPath({ path: '$..options.index', - json: schema.paths, + json: safePaths, resultType: 'pointer' - }).map(p=>{return p.split('/')[1]}) + }).map(p=>{ + + debug('\t\t','p',p) + return p + }) + //return p.split('.')[1]}) debug('\t\tindexed', indexed) let unique = JSONPath({ path: '$..options.unique', - json: schema.paths, + json: safePaths, resultType: 'pointer' }).map(p=>{ debug(typeof p) if(typeof p == 'string'){ - return p.split('/')[1] + return p.split('.')[1] } return p diff --git a/src/service/service-host.js b/src/service/service-host.js index 10033fa..6ebf54f 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -58,8 +58,9 @@ class ServiceHost { i2pSamHost = '127.0.0.1', i2pSamPort = 7656, i2pKey = null, - i2pForwardHost = 'localhost', + i2pForwardHost = '127.0.0.1', i2pForwardPort = null, + i2pOptions = null, wsEnabled = true, wsPort = null, wsUpgradePath = '/ws', @@ -146,6 +147,9 @@ class ServiceHost { forward: { host: i2pForwardHost ? i2pForwardHost : this.apiServerUri.hostname, port: i2pForwardPort ? i2pForwardPort : parseInt( this.apiServerUri.port ) + }, + session: { + options: i2pOptions } } } @@ -250,9 +254,7 @@ class ServiceHost { this.i2p = await SAM.createForward(this.i2pSettings) this.i2pUri = this.i2p.getB32Address() - this.i2pSettings.privateKey = null // clear no longer needed - - + this.i2pSettings.sam.privateKey = null // clear no longer needed this.i2p.on('error', this.reportI2pError.bind(this)) diff --git a/src/venue/bin/commands/venued-admin-add.js b/src/venue/bin/commands/venued-admin-add.js new file mode 100644 index 0000000..209e6fe --- /dev/null +++ b/src/venue/bin/commands/venued-admin-add.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminAdd extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminAdd.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin add' + } + + static get Definition(){ + return { + usage: `venued admin add [key-hash]`, + description: 'Add admin key', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + + let admins = (await config.read('admins')) || [] + + + let newAdmins = [] + + + for(let admin of parsed._.slice(2)){ + if(admins.indexOf(admin) != -1){ continue } + + newAdmins.push(admin) + } + + admins = admins.concat(newAdmins) + + await config.write('admins', admins) + await config.save() + + //console.log('admin added -', newAdmins) + + return {newAdmins} + } +} + +module.exports = VenuedAdminAdd \ No newline at end of file diff --git a/src/venue/bin/commands/venued-admin-list.js b/src/venue/bin/commands/venued-admin-list.js new file mode 100644 index 0000000..88d1c29 --- /dev/null +++ b/src/venue/bin/commands/venued-admin-list.js @@ -0,0 +1,69 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminList extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin list' + } + + static get Definition(){ + return { + usage: `venued admin list [key-hash]`, + description: 'Add admin key', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + let config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + let admins = (await config.read('admins')) || [] + + config=null + + return {admins} + } +} + +module.exports = VenuedAdminList \ No newline at end of file diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js new file mode 100644 index 0000000..8ebadd3 --- /dev/null +++ b/src/venue/bin/commands/venued-host.js @@ -0,0 +1,277 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.host') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + }, + l: { + alias: 'listen', + description: 'listen uri this can be an https:// with any local IP or file:/// to use a Unix socket', + default: 'https://0.0.0.0:3000' + }, + k: { + alias: 'ssl-key', + description: 'SSL key in pem format', + default: Path.join(DEFAULT_PATH, 'key.pem') + }, + c: { + alias: 'ssl-cert', + description: 'SSL Certificate in pem format', + default: Path.join(DEFAULT_PATH, 'cert.pem') + }, + A: { + alias: 'allow-ip', + description: 'IP to add to allow list', + multiple: true + }, + 'db-type': { + description: 'Type of database', + valid: ['tingo', 'loki', 'mongo', 'peer'], + default: 'tingo' + }, + 'db-uri': { + description: 'url to connect to database', + default: Path.join(DEFAULT_PATH, 'db/') + }, + 'db-peer': { + description: 'Peer database identity hash' + }, + 'service-code': { + description: 'path to service implementation', + default: Path.join(__dirname, '../../venue-service.js') + }, + 'service-build': { + description: 'path to service build', + default: Path.join(__dirname, '../../dataparty/@dataparty-venue.service.venue.json') + }, + 'full-errors': { + description: 'full server side error messages sent to client', + type:'boolean', + default: false + }, + i2p: { + description: 'Enable i2p hosting', + type: 'boolean', + default: false + }, + 'i2p-host':{ + default: '127.0.0.1' + }, + 'i2p-port': { + default: 7656 + }, + 'trust-proxy': { + type: 'boolean', + default: false + } +} + +function getPartyByType(type){ + if(type == 'tingo'){ + return Dataparty.TingoParty + } +} + +class VenuedHost extends CmdTree.Command { + constructor(context){ + super({...VenuedHost.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'host' + } + + static get Definition(){ + return { + usage: `venued host [options]`, + description: 'Start the hosting venued', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + /*if (!parsed.name){ + throw new CmdTree.Error.UsageError('no name provided') + }*/ + + + const ServiceCode = require(parsed['service-code']) + const ServiceBuild = require(parsed['service-build']) + const ServiceSchema = { + package: ServiceBuild.package, + ...ServiceBuild.schemas + } + + const PARTY = getPartyByType(parsed['db-type']) + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + //if(parsed['db-uri'][0] == '/'){ + config.touchDir( 'db' ) + //} + + const party = new PARTY({ + path: parsed['db-uri'], + model: ServiceSchema, + config: config, + noCache: false + }) + + const service = new ServiceCode( ServiceSchema.package, ServiceBuild ) + + debug('loaded service') + + debug('party db location', parsed['db-uri']) + + const CustomIpFilter = { + options: { + mode: 'allow', + //trustProxy: true + }, + ips: [ + '173.245.48.0/20', + '103.21.244.0/22', + '103.22.200.0/22', + '103.31.4.0/22', + '141.101.64.0/18', + '108.162.192.0/18', + '190.93.240.0/20', + '188.114.96.0/20', + '197.234.240.0/22', + '198.41.128.0/17', + '162.158.0.0/15', + '104.16.0.0/13', + '104.24.0.0/14', + '172.64.0.0/13', + '131.0.72.0/22', + '2400:cb00::/32', + '2606:4700::/32', + '2803:f800::/32', + '2405:b500::/32', + '2405:8100::/32', + '2a06:98c0::/29', + '2c0f:f248::/32', + '10.115.68.55/32' //! + ] + } + + if(parsed['allow-ip']){ + for(let ip of parsed['allow-ip']){ + CustomIpFilter.ips.push(ip) + } + } + + + const runner = new Dataparty.ServiceRunnerNode({ + party, service, + sendFullErrors: parsed['full-errors'], + useNative: false, + prefix: 'venue/' + }) + + + let runnerRouter = new Dataparty.RunnerRouter(runner) + + if(!fs.existsSync(parsed['ssl-key'])){ + execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', + {cwd: parsed.path} + ) + } + + const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') + const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') + + + + if(parsed.i2p && !await config.read('i2p.sam')){ + debug('i2p - creating key') + + const SAM = require('@diva.exchange/i2p-sam') + + const i2pSettings = { + sam: { + host: parsed['i2p-host'], + portTCP: parsed['i2p-port'], + }, + session: { + options: 'i2cp.leaseSetEncType=6,4' + } + } + + let i2p = await SAM.createLocalDestination(i2pSettings) + + + await Promise.all([ + config.write('i2p.address', i2p.address), + config.write('i2p.sam.publicKey', i2p.public), + config.write('i2p.sam.privateKey', i2p.private), + ]) + + await config.save() + } + + const host = new Dataparty.ServiceHost({ + runner: runnerRouter, + trust_proxy: parsed['trust-proxy/'], + wsEnabled: true, + ssl_key, ssl_cert, + listenUri: parsed.listen, + staticPath: Path.join(__dirname,'../../public'), + staticPrefix: '/venue/', + ipFilter: CustomIpFilter, + i2pEnabled: parsed.i2p, + i2pSamHost: parsed['i2p-host'], + i2pSamPort: parsed['i2p-port'], + i2pForwardHost: '127.0.0.1', + i2pForwardPort: 3000, + //i2pOptions: 'i2cp.leaseSetEncType=6,4', + i2pOptions: 'i2cp.leaseSetEncType=4,0', + i2pKey: await config.read('i2p.sam') + }) + + await party.start() + await runner.start() + await host.start() + + debug('started') + console.log('partying') + console.log(Path.join(__dirname,'../public')) + + this.context.exiting = false + + return {} + } +} + +module.exports = VenuedHost \ No newline at end of file diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js new file mode 100755 index 0000000..393955f --- /dev/null +++ b/src/venue/bin/venue.js @@ -0,0 +1,71 @@ +#!/usr/bin/env node + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree + + +const commandTree = new CommandTree({ usage: 'venue [command] \nVersion: ' + Pkg.version }) + +/*commandTree.addCommand(require('./project/project-init')) +commandTree.addCommand(require('./project/project-show')) +commandTree.addCommand(require('./project/project-mount')) +commandTree.addCommand(require('./developer/developer-add')) +commandTree.addCommand(require('./team/team-add')) +commandTree.addCommand(require('./cloud/cloud-add')) +commandTree.addCommand(require('./cloud/cloud-list')) +commandTree.addCommand(require('./package/package-add')) +commandTree.addCommand(require('./service/service-add'))*/ + + +/* +venued host --path /opt/venue +venued get version/identity/config +venued admin add/rm/list + +venue remote add --url [] --ws [] --identity <> +venue remote rm +venue remote switch + +venue package build --output [] +venue package deploy + +venue project build +venue project deploy +*/ + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + const output = await commandTree.run({context: { + + }}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}) diff --git a/src/venue/bin/venued.js b/src/venue/bin/venued.js new file mode 100755 index 0000000..a189251 --- /dev/null +++ b/src/venue/bin/venued.js @@ -0,0 +1,82 @@ +#!/usr/bin/env -S node --trace-warnings + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree + + +const commandTree = new CommandTree({ usage: 'venued [command] \nVersion: ' + Pkg.version }) + +commandTree.addCommand(require('./commands/venued-host')) +commandTree.addCommand(require('./commands/venued-admin-add')) +commandTree.addCommand(require('./commands/venued-admin-list')) + +/*commandTree.addCommand(require('./project/project-init')) +commandTree.addCommand(require('./project/project-show')) +commandTree.addCommand(require('./project/project-mount')) +commandTree.addCommand(require('./developer/developer-add')) +commandTree.addCommand(require('./team/team-add')) +commandTree.addCommand(require('./cloud/cloud-add')) +commandTree.addCommand(require('./cloud/cloud-list')) +commandTree.addCommand(require('./package/package-add')) +commandTree.addCommand(require('./service/service-add'))*/ + + +/* +venued host --path /opt/venue +venued get version/identity/config +venued admin add/rm/list + +venue remote add --url [] --ws [] --identity <> +venue remote rm +venue remote switch + +venue package build --output [] +venue package deploy + +venue project build +venue project deploy +*/ + +let context = { + exiting: true +} + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + const output = await commandTree.run({context}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index a205839..e7802bd 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -4,7 +4,7 @@ const debug = require('debug')('venue.venue_pkg') const ISchema = require('../../bouncer/ischema') -const Utils = ISchema.Utils +//const Utils = ISchema.Utils class VenuePkg extends ISchema { From e94db4e5fa7abdb443d877cc08335fd3738e0597 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 10 Jun 2026 07:27:19 +0000 Subject: [PATCH 16/50] moar commands --- src/service/service-host.js | 35 ++--- src/venue/bin/commands/venue-identity-gen.js | 85 ++++++++++++ src/venue/bin/commands/venue-identity-list.js | 56 ++++++++ src/venue/bin/commands/venue-identity-show.js | 85 ++++++++++++ src/venue/bin/commands/venue-remote-add.js | 94 +++++++++++++ src/venue/bin/commands/venue-remote-list.js | 71 ++++++++++ src/venue/bin/commands/venue-remote-show.js | 66 +++++++++ src/venue/bin/commands/venued-admin-list.js | 2 +- src/venue/bin/commands/venued-host.js | 11 +- src/venue/bin/venue.js | 126 ++++++++++++++---- src/venue/bin/venued.js | 2 +- src/venue/build.js | 1 + src/venue/schema/venue-project.js | 1 + 13 files changed, 593 insertions(+), 42 deletions(-) create mode 100644 src/venue/bin/commands/venue-identity-gen.js create mode 100644 src/venue/bin/commands/venue-identity-list.js create mode 100644 src/venue/bin/commands/venue-identity-show.js create mode 100644 src/venue/bin/commands/venue-remote-add.js create mode 100644 src/venue/bin/commands/venue-remote-list.js create mode 100644 src/venue/bin/commands/venue-remote-show.js diff --git a/src/service/service-host.js b/src/service/service-host.js index 6ebf54f..109edb7 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -250,27 +250,32 @@ class ServiceHost { if(this.i2pEnabled && this.i2p == null){ debug('starting i2p forward', this.i2pSettings) - const SAM = require('@diva.exchange/i2p-sam') - this.i2p = await SAM.createForward(this.i2pSettings) - this.i2pUri = this.i2p.getB32Address() - this.i2pSettings.sam.privateKey = null // clear no longer needed + async function setup_i2p(){ + const SAM = require('@diva.exchange/i2p-sam') - this.i2p.on('error', this.reportI2pError.bind(this)) + this.i2p = await SAM.createForward(this.i2pSettings) + this.i2pUri = this.i2p.getB32Address() + this.i2pSettings.sam.privateKey = null // clear no longer needed + this.i2p.on('error', this.reportI2pError.bind(this)) - this.i2p.on('close', ()=>{ - debug('i2p closed') - }) - this.i2p.on('data', (data)=>{ - debug('i2p data') - debug(data.toString()) - }) + this.i2p.on('close', ()=>{ + debug('i2p closed') + }) + + this.i2p.on('data', (data)=>{ + debug('i2p data') + debug(data.toString()) + }) + + debug('i2p started') + debug('\t', 'address', this.i2pUri) + debug('\t', 'key', this.i2p.getPublicKey()) + } - debug('i2p started') - debug('\t', 'address', this.i2pUri) - debug('\t', 'key', this.i2p.getPublicKey()) + setup_i2p() } if(this.mdnsEnabled && this.apiServer && this.apiServerUri.protocol != 'file:'){ diff --git a/src/venue/bin/commands/venue-identity-gen.js b/src/venue/bin/commands/venue-identity-gen.js new file mode 100644 index 0000000..6cb6d2b --- /dev/null +++ b/src/venue/bin/commands/venue-identity-gen.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-gen') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityGen extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityGen.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity gen' + } + + static get Definition(){ + return { + usage: `venue identity gen [name]`, + description: 'Create an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + if(await this.context.secureConfig.read('identity.'+keyName+'.phrase') && !parsed.force){ + throw new CmdTree.Error.UsageError('Key already exists!') + } + + const phrase = await dataparty_crypto.Routines.generateMnemonic() + + const password = parsed.nopassword ? null : await this.context.collectPassword() + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + await this.context.secureConfig.write('identity.'+keyName+'.phrase', phrase) + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityGen + + diff --git a/src/venue/bin/commands/venue-identity-list.js b/src/venue/bin/commands/venue-identity-list.js new file mode 100644 index 0000000..171fdf3 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-list.js @@ -0,0 +1,56 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueIdentityList extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity list' + } + + static get Definition(){ + return { + usage: `venue identity list [name]`, + description: 'List identity nicknames', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('identity')) + + return {names} + } +} + +module.exports = VenueIdentityList + + diff --git a/src/venue/bin/commands/venue-identity-show.js b/src/venue/bin/commands/venue-identity-show.js new file mode 100644 index 0000000..e289c74 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-show.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityShow extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity show' + } + + static get Definition(){ + return { + usage: `venue identity show [name]`, + description: 'Show an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exists!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityShow + + diff --git a/src/venue/bin/commands/venue-remote-add.js b/src/venue/bin/commands/venue-remote-add.js new file mode 100644 index 0000000..96a8093 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-add.js @@ -0,0 +1,94 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteAdd extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteAdd.Definition, context}) + } + + static get Command(){ + return 'remote add' + } + + static get Definition(){ + return { + usage: `venue remote add [name]`, + description: 'Add remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + if(await this.context.secureConfig.read('remote.'+remoteName) && !parsed.force){ + throw new CmdTree.Error.UsageError('Remote already exists!') + } + + const identityUrl = parsed.url+'/identity' + const versionUrl = parsed.url+'/version' + + const identity = await Dataparty.Comms.RestComms.HttpGet(identityUrl) + const version = await Dataparty.Comms.RestComms.HttpGet(versionUrl) + + const remote = { + url: parsed.url, + ws: parsed.ws, + i2p: parsed.i2p, + identity, version + } + + await this.context.secureConfig.write('remote.'+remoteName, remote) + + return { remote } + } +} + +module.exports = VenueRemoteAdd diff --git a/src/venue/bin/commands/venue-remote-list.js b/src/venue/bin/commands/venue-remote-list.js new file mode 100644 index 0000000..ed9b1fe --- /dev/null +++ b/src/venue/bin/commands/venue-remote-list.js @@ -0,0 +1,71 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteList extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteList.Definition, context}) + } + + static get Command(){ + return 'remote list' + } + + static get Definition(){ + return { + usage: `venue remote list [name]`, + description: 'List remote parties', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('remote')) + + return { names } + } +} + +module.exports = VenueRemoteList diff --git a/src/venue/bin/commands/venue-remote-show.js b/src/venue/bin/commands/venue-remote-show.js new file mode 100644 index 0000000..c8f1e63 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-show.js @@ -0,0 +1,66 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueRemoteShow extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote show' + } + + static get Definition(){ + return { + usage: `venue remote show [name]`, + description: 'Show a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + + return {remote} + } +} + +module.exports = VenueRemoteShow + + diff --git a/src/venue/bin/commands/venued-admin-list.js b/src/venue/bin/commands/venued-admin-list.js index 88d1c29..da0555e 100644 --- a/src/venue/bin/commands/venued-admin-list.js +++ b/src/venue/bin/commands/venued-admin-list.js @@ -41,7 +41,7 @@ class VenuedAdminList extends CmdTree.Command { static get Definition(){ return { usage: `venued admin list [key-hash]`, - description: 'Add admin key', + description: 'List admin keys', definition: DEFINITION } } diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 8ebadd3..c53d66f 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -266,11 +266,18 @@ class VenuedHost extends CmdTree.Command { debug('started') console.log('partying') - console.log(Path.join(__dirname,'../public')) + console.log('\t', parsed.listen) + + const i2pAddress = await config.read('i2p.address') + if(i2pAddress){ + console.log('\t', i2pAddress) + } + + //console.log(Path.join(__dirname,'../public')) this.context.exiting = false - return {} + return } } diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index 393955f..d99e4fc 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -3,36 +3,79 @@ const Pkg = require('../../../package.json') const debug = require('debug')('venue') const CommandTree = require('command-tree').CommandTree +const prompt = require('prompt') +const argon2 = require('argon2') +const OS = require('os') +const Path = require('path') + + +const Dataparty = require('../../../') const commandTree = new CommandTree({ usage: 'venue [command] \nVersion: ' + Pkg.version }) -/*commandTree.addCommand(require('./project/project-init')) -commandTree.addCommand(require('./project/project-show')) -commandTree.addCommand(require('./project/project-mount')) -commandTree.addCommand(require('./developer/developer-add')) -commandTree.addCommand(require('./team/team-add')) -commandTree.addCommand(require('./cloud/cloud-add')) -commandTree.addCommand(require('./cloud/cloud-list')) -commandTree.addCommand(require('./package/package-add')) -commandTree.addCommand(require('./service/service-add'))*/ +commandTree.addCommand(require('./commands/venue-identity-gen')) +commandTree.addCommand(require('./commands/venue-identity-list')) +commandTree.addCommand(require('./commands/venue-identity-show')) + +commandTree.addCommand(require('./commands/venue-remote-add')) +commandTree.addCommand(require('./commands/venue-remote-list')) +commandTree.addCommand(require('./commands/venue-remote-show')) + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = '.venue' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + +let secureConfig = null +async function collectPassword(info=''){ + let password = '' -/* -venued host --path /opt/venue -venued get version/identity/config -venued admin add/rm/list + while(1){ + let passes = await prompt.get({ + properties: { + password1: { + message: 'Set'+info+' password', + hidden: true + }, + password2: { + message: 'Confim'+info+' password', + hidden: true + } + } + }) -venue remote add --url [] --ws [] --identity <> -venue remote rm -venue remote switch + if(passes.password1 == passes.password2){ -venue package build --output [] -venue package deploy + password = passes.password1 + break + } + + console.log("passwords don't match") + } + + return password +} -venue project build -venue project deploy -*/ +async function onSetupRequired(){ + + console.log('setup-required') + + const password = await collectPassword(' keychain') + + await secureConfig.setPassword(password, { + created: Date.now() + }) + + console.log('password set') + + + await secureConfig.unlock(password) +} + +let context = { + exiting: true +} async function main(){ @@ -41,10 +84,42 @@ async function main(){ if(process.send){ process.send(commandTree.getHelp()) } return } + + + let config = new Dataparty.Config.JsonFileConfig({basePath: DEFAULT_PATH}) + secureConfig = new Dataparty.Config.SecureConfig({ + config, + timeoutMs: 60*1000*5, + argon: argon2 + }) + + secureConfig.on('setup-required', onSetupRequired) + + console.log('starting') + + await config.start() + await secureConfig.start() + + + if(await secureConfig.isInitialized() && secureConfig.isLocked()){ + + const {password} = await prompt.get({ + properties: { + password: { + message: 'Enter password', + hidden: true + } + }}) + + await secureConfig.unlock(password) + } + await secureConfig.waitForUnlocked('startup') + const output = await commandTree.run({context: { - + secureConfig, collectPassword, + ...context }}) if(output){ @@ -68,4 +143,9 @@ main().catch((error) => { }) } //process.exit() -}) +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) \ No newline at end of file diff --git a/src/venue/bin/venued.js b/src/venue/bin/venued.js index a189251..fbe37cd 100755 --- a/src/venue/bin/venued.js +++ b/src/venue/bin/venued.js @@ -1,4 +1,4 @@ -#!/usr/bin/env -S node --trace-warnings +#!/usr/bin/env node const Pkg = require('../../../package.json') const debug = require('debug')('venue') diff --git a/src/venue/build.js b/src/venue/build.js index e126d57..98c55ab 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -59,6 +59,7 @@ venue: { - default-config.json - public/ - package/NAME + - project/NAME - venue.json { projects: { diff --git a/src/venue/schema/venue-project.js b/src/venue/schema/venue-project.js index 51d77fa..9166252 100644 --- a/src/venue/schema/venue-project.js +++ b/src/venue/schema/venue-project.js @@ -17,6 +17,7 @@ class VenueProject extends ISchema { created: {type: Number, required: true}, changed: {type: Number}, venue: {type: String}, + name: {type: String}, domain: {type: String, index: true, unique: true}, // i2p enable? From b6e4ac74409bf403113607276b495f952b3535fd Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 10 Jun 2026 09:25:31 +0000 Subject: [PATCH 17/50] venue package build --- package.json | 1 + src/service/service-builder.js | 20 ++++- src/venue/bin/commands/pkg-build.js | 122 ++++++++++++++++++++++++++++ src/venue/bin/venue.js | 2 + src/venue/build.js | 4 +- 5 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 src/venue/bin/commands/pkg-build.js diff --git a/package.json b/package.json index 7b16dcd..88e0447 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", "fast-safe-stringify": "^2.1.1", + "find-up-json": "^2.0.5", "git-repo-info": "^2.1.1", "glob": "^13.0.6", "joi": "^18.2.1", diff --git a/src/service/service-builder.js b/src/service/service-builder.js index c78d3fe..8e5e2bb 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -52,7 +52,7 @@ module.exports = class ServiceBuilder { debug('compiling sources',this.service.sources) - await Promise.all([ + let results = await Promise.all([ this.compileMiddleware('pre'), this.compileMiddleware('post'), this.compileList('documents'), @@ -82,6 +82,7 @@ module.exports = class ServiceBuilder { } } + let files = [] if(writeFile){ const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.service.venue.json' @@ -100,11 +101,16 @@ module.exports = class ServiceBuilder { const compressedBrotli = zlib.brotliCompressSync(JSON.stringify(this.service.compiled, null,2)); console.log('Original:', JSON.stringify(this.service.compiled, null,2).length, 'bytes'); - console.log('Gzip:', compressed.length, 'bytes'); - console.log('Brotli:', compressedBrotli.length, 'bytes'); + console.log('Gzip:', compressed.length, 'bytes') + console.log('Brotli:', compressedBrotli.length, 'bytes') + + let tarFile = results[ results.length - 1 ] + files.push(buildOutput) + files.push(schemaOutput) + if(tarFile){files.push(tarFile)} } - return this.service.compiled + return {build: this.service.compiled, files} } @@ -384,6 +390,8 @@ module.exports = class ServiceBuilder { async compressFiles(outputPath, writeFile){ + if(!this.service.sources.files){ return } + let fileMap={} let files = this.service.sources.files.map(file=>{ @@ -398,6 +406,8 @@ module.exports = class ServiceBuilder { return hash }) + if(!files || files.length < 1){ return } + const tarFileName = this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz' const tarPath = Path.join(outputPath, tarFileName) @@ -420,5 +430,7 @@ module.exports = class ServiceBuilder { files: fileMap } } + + return tarPath } } \ No newline at end of file diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js new file mode 100644 index 0000000..11a9db7 --- /dev/null +++ b/src/venue/bin/commands/pkg-build.js @@ -0,0 +1,122 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.package-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') +const findUp = require('find-up-json').default + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'package name' + }, + version: { + description: 'package version' + } +} + + +class VenuePackageBuild extends CmdTree.Command { + constructor(context){ + super({...VenuePackageBuild.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'package build' + } + + static get Definition(){ + return { + usage: `venue package build [service-code.js]`, + description: 'Build a package', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply service code') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const serviceClassPath = Path.resolve(parsed._[2]) + + let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content + + const ServiceClass = require( serviceClassPath ) + + const service = new ServiceClass({ + name: parsed.name ? parsed.name : pkgJson.name, + version: parsed.version ? parsed.version : pkgJson.version, + }) + + await mkdirp(parsed.output) + + const builder = new Dataparty.ServiceBuilder(service) + const build = await builder.compile(parsed.output, true, key) + + return {files: build.files} + } +} + +module.exports = VenuePackageBuild + + diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index d99e4fc..fc715db 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -22,6 +22,8 @@ commandTree.addCommand(require('./commands/venue-remote-add')) commandTree.addCommand(require('./commands/venue-remote-list')) commandTree.addCommand(require('./commands/venue-remote-show')) +commandTree.addCommand(require('./commands/pkg-build')) + const HOMEDIR = OS.homedir() const DEFAULT_FOLDER = '.venue' const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) diff --git a/src/venue/build.js b/src/venue/build.js index 98c55ab..04e0416 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -174,11 +174,9 @@ async function main(){ await pushService( party.privateIdentity, build, staticTar ) - - } main().catch(err=>{ console.error('CRASH') console.error(err) -}) \ No newline at end of file +}) From 690a5428f0522f868b08562632f5b8d56b27893f Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 12 Jun 2026 01:26:53 +0000 Subject: [PATCH 18/50] i2p tweaks --- scripts/install-i2p-ubuntu-24.sh | 6 ++++++ src/service/service-host.js | 7 +++++-- src/venue/bin/commands/venued-host.js | 9 +++++---- 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100755 scripts/install-i2p-ubuntu-24.sh diff --git a/scripts/install-i2p-ubuntu-24.sh b/scripts/install-i2p-ubuntu-24.sh new file mode 100755 index 0000000..55019f1 --- /dev/null +++ b/scripts/install-i2p-ubuntu-24.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo add-apt-repository ppa:purplei2p/i2pd +sudo apt update +sudo apt install --yes i2pd + diff --git a/src/service/service-host.js b/src/service/service-host.js index 109edb7..85bb43e 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -142,9 +142,12 @@ class ServiceHost { host: i2pSamHost, portTCP: i2pSamPort, publicKey: reach(i2pKey, 'publicKey'), - privateKey: reach(i2pKey, 'privateKey') + privateKey: reach(i2pKey, 'privateKey'), + versionMin: '3.1', + versionMax: '3.3' }, forward: { + silent: true, host: i2pForwardHost ? i2pForwardHost : this.apiServerUri.hostname, port: i2pForwardPort ? i2pForwardPort : parseInt( this.apiServerUri.port ) }, @@ -275,7 +278,7 @@ class ServiceHost { debug('\t', 'key', this.i2p.getPublicKey()) } - setup_i2p() + setup_i2p.bind(this)() } if(this.mdnsEnabled && this.apiServer && this.apiServerUri.protocol != 'file:'){ diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index c53d66f..2ef584c 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -226,6 +226,7 @@ class VenuedHost extends CmdTree.Command { }, session: { options: 'i2cp.leaseSetEncType=6,4' + //options: 'i2cp.leaseSetEncType=4' } } @@ -254,9 +255,9 @@ class VenuedHost extends CmdTree.Command { i2pSamHost: parsed['i2p-host'], i2pSamPort: parsed['i2p-port'], i2pForwardHost: '127.0.0.1', - i2pForwardPort: 3000, - //i2pOptions: 'i2cp.leaseSetEncType=6,4', - i2pOptions: 'i2cp.leaseSetEncType=4,0', + i2pForwardPort: '3000', + i2pOptions: 'i2cp.leaseSetEncType=6,4', + //i2pOptions: 'i2cp.leaseSetEncType=4', i2pKey: await config.read('i2p.sam') }) @@ -281,4 +282,4 @@ class VenuedHost extends CmdTree.Command { } } -module.exports = VenuedHost \ No newline at end of file +module.exports = VenuedHost From df68ba395e974d86d3c976faf53c9abc62b88920 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 13 Jun 2026 02:17:04 +0000 Subject: [PATCH 19/50] initial create project --- src/venue/endpoints/create-project.js | 308 ++++++++++++++++++++++++++ src/venue/schema/venue-project.js | 83 ------- src/venue/schema/venue_project.js | 99 +++++++++ src/venue/venue-service.js | 2 + 4 files changed, 409 insertions(+), 83 deletions(-) create mode 100644 src/venue/endpoints/create-project.js delete mode 100644 src/venue/schema/venue-project.js create mode 100644 src/venue/schema/venue_project.js diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js new file mode 100644 index 0000000..ab0d229 --- /dev/null +++ b/src/venue/endpoints/create-project.js @@ -0,0 +1,308 @@ +const fs = require('fs') +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Message, Routines, Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-project') +const zlib = require('zlib') + +const IEndpoint = require('../../service/iendpoint') + +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + +module.exports = class CreateProjectEndpoint extends IEndpoint { + + static get Name(){ + return 'create-project' + } + + + static get Description(){ + return 'Create venue project' + } + + + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + ephemeral_session: true, + validate: Joi.object().keys({ + project: Joi.object().keys({ + owner: Joi.string(), + created: Joi.number(), + changed: Joi.number(), + + name: Joi.string(), + venue: Joi.string(), + domain: Joi.string(), + + i2p: Joi.object().keys({ + address: Joi.string(), + public: Joi.string(), + securePrivate: Joi.string() + }), + party: Joi.array().items(Joi.object().keys({ + name: Joi.string(), + type: Joi.string(), + tingo: { path: Joi.string() }, + loki: { path: Joi.string() }, + peer: { + venue: Joi.string(), + remoteIdentity: Joi.string() + }, + key: { + hash: Joi.string(), + securePrivate: Joi.string() + }, + settings: { noCache: Joi.string() }, + defaultConfig: Joi.string() + })), + routes: Joi.array().items(Joi.object().keys({ + prefix: Joi.string(), + party: Joi.string(), + package: Joi.object().keys({ + owner: Joi.string(), + name: Joi.string().required(), + version: Joi.string(), + branch: Joi.string(), + hash: Joi.string() + }), + settings: { + sendFullErrors: Joi.boolean().required(), + useNative: Joi.boolean().required() + } + })), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + tar: Joi.string(), + hash: Joi.string().required(), + size: Joi.number().required(), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + hash: Joi.string().required(), + size: Joi.number().required() + })) + })) + }).required(), + signatures: Joi.object().pattern(Joi.string(), Joi.string()).required(), + staticTar: Joi.any().custom(typedArraySchema) + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + if(ctx.party.identity.key.hash != ctx.input.project.hash){ + throw new Error('project venue does not match this host') + } + + let {signatures, ...projectWithoutSig} = ctx.input.project + + const projectOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.project.owner ) + .exec() + )[0] + + if(!projectOwnerIdentityDoc){ + throw new Error('project owner not authorized') + } + + debug('found project owner', projectOwnerIdentityDoc.hash) + + const projectOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: projectOwnerIdentityDoc.data.type, + hash: projectOwnerIdentityDoc.data.hash, + public: projectOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.project.owner]) + + let signedProjectMsg = new Message({ + msg: projectWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') + + await signedProjectMsg.assertVerified(projectOwnerIdentity, true) + + debug('verified package signature') + + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const safeFileName = ctx.input.project.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' + + const projectFiles = ctx.input.project.files[tarFileName] + + if(projectFiles && projectFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match project definition") + } + + debug('verified staticTar') + + debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) + + const projectBSON = Routines.BSON.serializeBSONWithoutOptimiser(projectWithoutSig) + + const projectHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + projectBSON + ) + ) + + const safeProjectHash = projectHash.replace(/\//g, "-") + + debug('\t'+'hash', projectHash) + + const projectWorkspace = 'projects'+safeFileName+'/'+ctx.input.project.version+'/'+safeProjectHash + + const config = ctx.party.config + + const workspacePath = await config.touchDir(projectWorkspace) + + + debug('\t'+'workspace - local', projectWorkspace) + debug('\t'+'workspace - global', workspacePath) + + + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.project)) + + const project = ctx.input.project + const projectId = project.name + '@' + project.version + debug('addProject', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_project') + .where('project.name').equals(project.name) + .where('project.version').equals(project.version) + .where('hash').equals(projectHash) + .exec())[0] + + + if(!projectDoc){ + debug('creating project') + + const {owner, ...pkgWithoutOwner} = project.package + + projectDoc = await ctx.party.createDocument('venue_pkg', { + owner: project.owner, + created: Date.now(), + hash: projectHash, + workspace: workspacePath, + project: ctx.input.project, + signatures: ctx.input.signatures + }) + + debug('project created') + } else { + debug('need to update project?') + } + + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.project.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) + + // verify build signature + + //ctx.input. + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + + const compiledSrv = JSON.parse(ctx.input.service) + const projectId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!projectDoc){ + debug('creating service') + projectDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(projectDoc.data) + await projectDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(projectDoc.data) + + debug('saving doc') + + + await projectDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + }*/ + + return {done: true} + + //return {srv:projectDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/schema/venue-project.js b/src/venue/schema/venue-project.js deleted file mode 100644 index 9166252..0000000 --- a/src/venue/schema/venue-project.js +++ /dev/null @@ -1,83 +0,0 @@ -'use strict' - -const debug = require('debug')('venue.venue_project') - -const ISchema = require('../../bouncer/ischema') - -const Utils = ISchema.Utils - - -class VenueProject extends ISchema { - - static get Type () { return 'venue_project' } - - static get Schema(){ - return { - owner: {type: String, required: true, index: true}, //public_key.key.hash - created: {type: Number, required: true}, - changed: {type: Number}, - venue: {type: String}, - name: {type: String}, - domain: {type: String, index: true, unique: true}, - - // i2p enable? - // standalone hosting? - - workspace: {type: String, required: true}, - //hash: {type: String, required: true, index: true}, - party: [{ - name: String, - type: {type: String, enum: ['TingoParty', 'LokiParty', 'PeerParty', 'MongoParty']}, - tingo: { - path: String - }, - loki: { - path: String - }, - peer: { - venue: String, - remoteIdentity: String - }, - key: { - hash: String, - securePrivate: String - }, - settings: { - noCache: Boolean, - }, - defaultConfig: Object - }], - routes: [{ - prefix: String, - party: String, - package: { - owner: String, - name: {type: String, required: true, index: true}, - version: String, - branch: String, - hash: String, - }, - settings: { - sendFullErrors: {type: Boolean, required: true}, - useNative: {type: Boolean, required: true}, - } - }] - } - } - - static setupSchema(schema){ - //schema.index({ 'package.name': 1 }, {unique: true}) - return schema - } - - static permissions (context) { - return { - read: false, - new: false, - change: false - } - } -} - - -module.exports = VenueProject \ No newline at end of file diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js new file mode 100644 index 0000000..f96d08e --- /dev/null +++ b/src/venue/schema/venue_project.js @@ -0,0 +1,99 @@ +'use strict' + +const debug = require('debug')('venue.venue_project') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenueProject extends ISchema { + + static get Type () { return 'venue_project' } + + static get Schema(){ + return { + owner: {type: String, required: true}, + created: {type: Number, required: true}, + workspace: {type: String, required: true}, + hash: {type: String, required: true, index: true}, + + enabled: {type: Boolean}, + + + project: { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + + name: {type: String, required: true}, + version: {type: String, required: true}, + venue: {type: String}, + domain: {type: String, index: true, unique: true}, + + i2p: { + address: String, + public: String, + securePrivate: String + }, + + party: [{ + name: String, + type: {type: String, enum: ['tingo', 'loki', 'peer', 'mongo']}, + tingo: { + path: String + }, + loki: { + path: String + }, + peer: { + venue: String, + remoteIdentity: String + }, + key: { + hash: String, + securePrivate: String + }, + settings: { + noCache: Boolean, + }, + defaultConfig: String + }], + routes: [{ + prefix: String, + party: String, + package: { + owner: String, + name: {type: String, required: true}, + version: String, + branch: String, + hash: String, + }, + settings: { + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + } + }], + files: Object + }, + signatures: {type: Object, required: true} + + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenueProject \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index a304e47..815ea87 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -15,6 +15,7 @@ class VenueService extends DatapartySrv.IService { //builder.addSchema(Path.join(__dirname, './schema/public-key.js')) //builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) + builder.addSchema(Path.join(__dirname, './schema/venue_project.js')) builder.addSchema(DatapartySrv.schema_paths.public_key) builder.addSchema(DatapartySrv.schema_paths.session_key) @@ -35,6 +36,7 @@ class VenueService extends DatapartySrv.IService { //builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) From 45cfb172f5e5c92a4c0c6be4672e941e0e7c6b29 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 13 Jun 2026 02:18:48 +0000 Subject: [PATCH 20/50] install i2p script ubuntu --- scripts/install-i2p-ubuntu-24.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 scripts/install-i2p-ubuntu-24.sh diff --git a/scripts/install-i2p-ubuntu-24.sh b/scripts/install-i2p-ubuntu-24.sh new file mode 100755 index 0000000..55019f1 --- /dev/null +++ b/scripts/install-i2p-ubuntu-24.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo add-apt-repository ppa:purplei2p/i2pd +sudo apt update +sudo apt install --yes i2pd + From 2fa54e801678eabfeb59ceb14585772c640288f7 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 13 Jun 2026 02:46:27 +0000 Subject: [PATCH 21/50] clean up regression in schema generation --- src/service/service-builder.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 8e5e2bb..0598e4c 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -197,8 +197,8 @@ module.exports = class ServiceBuilder { resultType: 'pointer' }).map(p=>{ - debug('\t\t','p',p) - return p + debug('\t\t','indexed p',p) + return p.replace('/options/index', '').replace('/','') }) //return p.split('.')[1]}) @@ -209,9 +209,10 @@ module.exports = class ServiceBuilder { json: safePaths, resultType: 'pointer' }).map(p=>{ - debug(typeof p) + debug(typeof p, 'unique', p) if(typeof p == 'string'){ - return p.split('.')[1] + let filteredP = p.replace('/options/unique', '').replace('/','') + return filteredP } return p From b68cfb700433ce54b434913f0ecc72ce2e14762f Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 01:58:22 +0000 Subject: [PATCH 22/50] move sig --- src/venue/endpoints/create-project.js | 15 +++++++-------- src/venue/schema/venue_project.js | 6 +++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index ab0d229..a6b0739 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -36,12 +36,12 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { ephemeral_session: true, validate: Joi.object().keys({ project: Joi.object().keys({ - owner: Joi.string(), + owner: Joi.string().required(), created: Joi.number(), changed: Joi.number(), - name: Joi.string(), - venue: Joi.string(), + name: Joi.string().required(), + venue: Joi.string().required(), domain: Joi.string(), i2p: Joi.object().keys({ @@ -51,7 +51,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { }), party: Joi.array().items(Joi.object().keys({ name: Joi.string(), - type: Joi.string(), + type: Joi.string().required(), tingo: { path: Joi.string() }, loki: { path: Joi.string() }, peer: { @@ -88,9 +88,9 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { hash: Joi.string().required(), size: Joi.number().required() })) - })) + })), + signatures: Joi.object().pattern(Joi.string(), Joi.string()).required() }).required(), - signatures: Joi.object().pattern(Joi.string(), Joi.string()).required(), staticTar: Joi.any().custom(typedArraySchema) }) }, @@ -213,7 +213,6 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { hash: projectHash, workspace: workspacePath, project: ctx.input.project, - signatures: ctx.input.signatures }) debug('project created') @@ -233,7 +232,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { fs.writeFileSync( Path.join(workspacePath, safeFileName+'.project.venue.json'), - JSON.stringify(ctx.input.build, null, 2) + JSON.stringify(ctx.input.project, null, 2) ) // verify build signature diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index f96d08e..fd610c8 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -74,9 +74,9 @@ class VenueProject extends ISchema { useNative: {type: Boolean, required: true}, } }], - files: Object - }, - signatures: {type: Object, required: true} + files: Object, + signatures: {type: Object, required: true} + } } } From 2159b3b4b0543607218d5f962a97e7e5478ac379 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 04:06:04 +0000 Subject: [PATCH 23/50] origin router fork --- package.json | 2 +- src/venue/endpoints/create-project.js | 1 + src/venue/schema/venue_project.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 88e0447..671ac7f 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "node-mocks-http": "^1.12.1", "node-object-hash": "^3.0.0", "node-persist": "^3.0.1", - "origin-router": "^1.6.4", + "origin-router": "https://github.com/datapartyjs/origin-router.git", "parse-url": "^5.0.1", "promisfy": "^1.2.0", "roslib": "^1.3.0", diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index a6b0739..2216f9d 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -41,6 +41,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { changed: Joi.number(), name: Joi.string().required(), + version: Joi.string().required(), venue: Joi.string().required(), domain: Joi.string(), diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index fd610c8..e7e4f30 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -26,7 +26,7 @@ class VenueProject extends ISchema { created: {type: Number, required: true}, changed: {type: Number}, - name: {type: String, required: true}, + name: {type: String, required: true, index: true}, version: {type: String, required: true}, venue: {type: String}, domain: {type: String, index: true, unique: true}, From 1ee631920cf9b3be2be75ed90a79f7a569fcd083 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 05:11:47 +0000 Subject: [PATCH 24/50] ephemeral-client --- src/party/index-browser.js | 3 +- src/party/index-embedded.js | 3 +- src/party/index.js | 3 +- src/party/peer/ephemeral-client.js | 223 ++++++++++++++++++++++++ src/venue/bin/commands/project-build.js | 141 +++++++++++++++ src/venue/endpoints/create-project.js | 1 - src/venue/schema/venue_project.js | 4 +- 7 files changed, 372 insertions(+), 6 deletions(-) create mode 100644 src/party/peer/ephemeral-client.js create mode 100644 src/venue/bin/commands/project-build.js diff --git a/src/party/index-browser.js b/src/party/index-browser.js index cf40f10..b084914 100644 --- a/src/party/index-browser.js +++ b/src/party/index-browser.js @@ -7,6 +7,7 @@ const ZangoParty = require('./local/zango-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') const LokiDb = require('../bouncer/db/loki-db') @@ -15,5 +16,5 @@ module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, ZangoParty, PeerParty, - LokiDb, MatchMakerClient + LokiDb, EphemeralClient, MatchMakerClient } diff --git a/src/party/index-embedded.js b/src/party/index-embedded.js index 422f737..ba36e58 100644 --- a/src/party/index-embedded.js +++ b/src/party/index-embedded.js @@ -7,11 +7,12 @@ const TingoParty = require('./local/tingo-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, PeerParty, - TingoParty, MatchMakerClient + TingoParty, EphemeralClient, MatchMakerClient } \ No newline at end of file diff --git a/src/party/index.js b/src/party/index.js index e362eff..af6f1cc 100644 --- a/src/party/index.js +++ b/src/party/index.js @@ -9,4 +9,5 @@ exports.MongoParty = require('./mongo/mongo-party') exports.IDocument = require('./idocument') exports.DocumentFactory = require('./document-factory') exports.CloudDocument = require('./cloud/cloud-document') -exports.MatchMakerClient = require('./peer/match-maker-client') \ No newline at end of file +exports.EphemeralClient = require('./peer/ephemeral-client') +exports.MatchMakerClient = require('./peer/match-maker-client') diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js new file mode 100644 index 0000000..9cf9be6 --- /dev/null +++ b/src/party/peer/ephemeral-client.js @@ -0,0 +1,223 @@ +const EventEmitter = require('eventemitter3') + +const debug = require('debug')('dataparty.ephemeral-client') + +const dataparty_crypto = require('@dataparty/crypto') +const LokiParty = require('../local/loki-party') +const PeerParty = require('./peer-party') +const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') +const WebsocketComms = require('../../comms/websocket-comms') + +class EphemeralClient extends EventEmitter { + constructor({identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + + super() + + this.contacts = contacts + this.sessionKey = null + this.identity = identity + this.wsParty = null + this.restParty = null + + if(typeof urlOrParty == 'string'){ + this.restUrl = urlOrParty + this.restParty = null + } else { + this.restParty = urlOrParty + } + + if(typeof wsUrlOrParty == 'string'){ + this.wsUrl = wsUrlOrParty + this.wsParty = null + } else { + this.wsParty = wsUrlOrParty + } + } + + + + async start(){ + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + if(!this.restParty){ + let config = new MemoryConfig({ + basePath:'ephemeral-client', + cloud: { + uri: this.restUrl + } + }) + + this.restParty = new LokiParty({ + path: 'ephemeral-client', + dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), + config + }) + + await this.restParty.setIdentity(this.sessionKey) + + debug('starting restParty') + await this.restParty.start() + + if(!this.restParty.comms){ + this.restParty.comms = new RestComms({ + party:this.restParty, + config: this.restParty.config + }) + + this.restParty.comms.sessionId = this.sessionKey.key.hash + } + + await this.announcePublicKeys() + } + + if(!this.wsParty && this.wsUrl){ + this.wsParty = new PeerParty({ + comms: new WebsocketComms({ + uri: this.wsUrl, + discoverRemoteIdentity: false, + remoteIdentity: await this.restParty.comms.getServiceIdentity(), + session: this.sessionKey.key.hash + }), + config: this.restParty.config + }) + + await this.wsParty.start() + + debug('starting wsParty') + await this.wsParty.start() + debug('waiting for websocket authorization') + await this.wsParty.comms.authorized() + } + + } + + + + async announcePublicKeys(type='guest', callPath='key/announce'){ + + let currentActor = this.identity + + const announceData = { + annoucement: { + type, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + debug('announcePublicKeys', announceData) + + const announceResult = await this.restParty.comms.call(callPath, announceData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: false + }) + + if(announceResult.done != true){ + throw new Error('annoucement request failed - '+callPath) + } + } + + + async lookupPublicKey(hash){ + debug('lookupPublicKey - hash:', hash) + + if(hash == this.identity.key.hash){ + return this.identity + } + + if(this.contacts){ + return await this.contacts.lookupPublicKey(hash) + } + + const lookupData = { hash } + + const lookupResult = await this.wsParty.comms.call('key/lookup', lookupData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + if(!lookupResult.done){ + return null + } + + debug('lookup result -', lookupResult) + + const identity = new dataparty_crypto.Identity({ + key: lookupResult.public_key + }) + + return identity + } + + async createShortCode(use_limit=3, expiry){ + debug('createShortCode') + + const request = { + use_limit, + expiry: !expiry ? Date.now()+24*60*60*3 : expiry + } + + const result = await this.wsParty.comms.call('short-code/create', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('createShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } + + async lookupPublicKeyByShortCode( code ){ + debug('lookupPublicKeyByShortCode') + + const request = { code } + + const result = await this.wsParty.comms.call('short-code/lookup', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('lookupPublicKeyByShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } +} + +module.exports = EphemeralClient diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js new file mode 100644 index 0000000..038f218 --- /dev/null +++ b/src/venue/bin/commands/project-build.js @@ -0,0 +1,141 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.project-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') +const findUp = require('find-up-json').default + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') +const Joi = require('joi') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'project name' + }, + version: { + description: 'project version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + } +} + + +class VenueProjectBuild extends CmdTree.Command { + constructor(context){ + super({...VenueProjectBuild.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'project build' + } + + static get Definition(){ + return { + usage: `venue project build [project.json]`, + description: 'Build a project', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply project json') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const projectJsonPath = Path.resolve(parsed._[2]) + + let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content + + let projectJson = require( projectJsonPath ) + + + const remoteName = parsed.remote || projectJson.venue + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + if(!remote){ + throw new CmdTree.Error.UsageError('Invalid remote ['+remoteName+']') + } + + const project = { + name: parsed.name ? parsed.name : projectJson.name, + version: parsed.version ? parsed.version : projectJson.version, + owner: key.id, + created: Date.now(), + + venue: remote.identity.key.hash, + domain: projectJson.domain, + + } + + await mkdirp(parsed.output) + + const builder = new Dataparty.ServiceBuilder(service) + const build = await builder.compile(parsed.output, true, key) + + return {files: build.files} + } +} + +module.exports = VenueProjectBuild + + diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index 2216f9d..c763206 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -38,7 +38,6 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { project: Joi.object().keys({ owner: Joi.string().required(), created: Joi.number(), - changed: Joi.number(), name: Joi.string().required(), version: Joi.string().required(), diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index e7e4f30..0dfa83e 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -15,6 +15,7 @@ class VenueProject extends ISchema { return { owner: {type: String, required: true}, created: {type: Number, required: true}, + changed: {type: Number, required: true}, workspace: {type: String, required: true}, hash: {type: String, required: true, index: true}, @@ -24,10 +25,9 @@ class VenueProject extends ISchema { project: { owner: {type: String, required: true, index: true}, //public_key.key.hash created: {type: Number, required: true}, - changed: {type: Number}, name: {type: String, required: true, index: true}, - version: {type: String, required: true}, + version: {type: String, required: true, index: true}, venue: {type: String}, domain: {type: String, index: true, unique: true}, From 9ec28a0bfe7bbc5b5fe415dc7f0e4593c1dec3d3 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 15:45:16 +0000 Subject: [PATCH 25/50] autoreconnect --- src/comms/isocket-comms.js | 1 + src/comms/peer-comms.js | 10 +- src/comms/rest-comms.js | 2 +- src/comms/websocket-comms.js | 36 ++++++- src/comms/websocket-shim.js | 3 +- src/party/peer/ephemeral-client.js | 77 ++++++++++++++- src/service/endpoints/key-announce.js | 3 +- src/venue/bin/commands/venue-remote-check.js | 99 ++++++++++++++++++++ src/venue/bin/venue.js | 11 ++- 9 files changed, 229 insertions(+), 13 deletions(-) create mode 100644 src/venue/bin/commands/venue-remote-check.js diff --git a/src/comms/isocket-comms.js b/src/comms/isocket-comms.js index 94070ae..cfe90fc 100644 --- a/src/comms/isocket-comms.js +++ b/src/comms/isocket-comms.js @@ -71,6 +71,7 @@ class ISocketComms extends EventEmitter { this.connected = false debug('Server closed connection') this.emit('close') + this.emit('server-close') } onopen(){ diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index b20fd16..65b9438 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -54,6 +54,8 @@ class PeerComms extends ISocketComms { this.uuid = uuidv4() this.socket = socket || null + this.stopped = false + //this.auto_reconnect = !socket && !host this.host = host //! Is comms host\ this.oncall = null @@ -207,7 +209,7 @@ class PeerComms extends ISocketComms { await this.socketInit() } - this.socket.on('close', this.stop.bind(this)) + this.socket.on('close', this.socketStop.bind(this)) if(this.host){ debug('host mode comms') @@ -226,7 +228,13 @@ class PeerComms extends ISocketComms { } } + socketStop(){ + debug('socket stop') + this.close() + } + async stop(){ + this.stopped = true debug('stop') this.close() } diff --git a/src/comms/rest-comms.js b/src/comms/rest-comms.js index cac3a67..485e345 100644 --- a/src/comms/rest-comms.js +++ b/src/comms/rest-comms.js @@ -207,7 +207,7 @@ class RestComms extends EventEmitter { const serverIdentity = await RestComms.HttpGet(this.uri + `${this.uriPrefix}identity`) debug('server identity - ', serverIdentity) - this.remoteIdentity = new dataparty_crypto.Identity(serverIdentity) + this.remoteIdentity = dataparty_crypto.Identity.fromJSON(serverIdentity) } return this.remoteIdentity diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index d91d690..d606cc2 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -14,11 +14,12 @@ const WebsocketShim = require('./websocket-shim') * @see https://en.wikipedia.org/wiki/WebSocket */ class WebsocketComms extends PeerComms { - constructor({uri, connection, remoteIdentity, host, party, ...options}){ + constructor({uri, connection, timeout=10000, remoteIdentity, host, party, ...options}){ super({remoteIdentity, host, party, ...options}) this.uri = uri this.connection = connection + this.timeout = timeout debug('starting host=',host, ' uuid=', this.uuid, ' uri=', this.uri) @@ -34,13 +35,44 @@ class WebsocketComms extends PeerComms { async socketInit(){ debug('init') - + let isNewConnection = false + if(!this.host && !this.connection){ debug('opening client connection to',this.uri) this.connection = new WebSocket(this.uri) + + isNewConnection = true } this.socket = new WebsocketShim(this.connection) + + if(isNewConnection){ + + //await new Promise((resolve,reject)=>{ + const timer = setTimeout(() => { + debug('websocket timeout') + this.connection.close() + this.emit('timeout') + //reject(new Error("WebSocket connection timeout")); + }, this.timeout); + + this.socket.once('connect', () => { + debug('websocket opened') + clearTimeout(timer); + //resolve(); + }) + + this.socket.once('error',(error) => { + debug('websocket error', error) + clearTimeout(timer) + this.emit('error', error) + //this.connection.close() + //reject(error); + }) + //}) + } + + } } diff --git a/src/comms/websocket-shim.js b/src/comms/websocket-shim.js index 1f8b2ec..1a6e72a 100644 --- a/src/comms/websocket-shim.js +++ b/src/comms/websocket-shim.js @@ -19,7 +19,7 @@ class WebsocketShim extends EventEmitter { } this.conn.onclose = (event) => { - debug('onclose', event) + debug('onclose', event, event.code, event.reason) this.emit('close', event) } @@ -39,6 +39,7 @@ class WebsocketShim extends EventEmitter { } destroy(){ + if(this.conn && this.conn.terminate) this.conn.terminate() } diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 9cf9be6..c2fd90c 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -10,15 +10,19 @@ const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') class EphemeralClient extends EventEmitter { - constructor({identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + constructor({identity, role='guest', autoreconnect=true, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ super() this.contacts = contacts this.sessionKey = null this.identity = identity + this.role = role || 'guest' this.wsParty = null this.restParty = null + this.autoreconnect = autoreconnect + + this.reconnectTimer = null if(typeof urlOrParty == 'string'){ this.restUrl = urlOrParty @@ -33,6 +37,9 @@ class EphemeralClient extends EventEmitter { } else { this.wsParty = wsUrlOrParty } + + this.reconnect_last_attempt = null + this.reconnect_tries = 0 } @@ -72,6 +79,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ + this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -82,6 +90,10 @@ class EphemeralClient extends EventEmitter { config: this.restParty.config }) + this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) + this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) + this.wsParty.comms.on('error', this.handleWsClose.bind(this)) + await this.wsParty.start() debug('starting wsParty') @@ -92,15 +104,74 @@ class EphemeralClient extends EventEmitter { } + async handleWsClose(){ + + let stopped = this.wsParty.comms.stopped + + const sleepTime = Math.min(9000*this.reconnect_tries, 20*1000) + debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') + + if(!this.reconnectTimer){ + this.reconnectTimer = setTimeout( + this.doReconnect.bind(this), + sleepTime + ) + } + } + + async doReconnect(){ + let stopped = this.wsParty.comms.stopped + + this.reconnectTimer = null + + if(stopped || !this.wsParty || !this.autoreconnect){ debug('skip reconnect'); return } + + debug('doing ws reconnect...') + + this.reconnect_last_attempt = Date.now() + this.reconnect_tries++ + + try{ + this.wsParty.comms = new WebsocketComms({ + uri: this.wsUrl, + discoverRemoteIdentity: false, + remoteIdentity: await this.restParty.comms.getServiceIdentity(), + session: this.sessionKey.key.hash + }) + + this.wsParty.comms.party = this.wsParty + + this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) + this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) + this.wsParty.comms.on('error', this.handleWsClose.bind(this)) + + debug('restarting websocket') + await this.wsParty.comms.start() + debug('waiting for websocket authorization') + await this.wsParty.comms.authorized() + + + debug('connected and authorized') + + this.reconnect_last_attempt = null + this.reconnect_tries = 0 + + } catch(err){ + debug('reconnect error', err) + + + this.handleWsClose() + } + } - async announcePublicKeys(type='guest', callPath='key/announce'){ + async announcePublicKeys(callPath='key/announce'){ let currentActor = this.identity const announceData = { annoucement: { - type, + role: this.role, created: Date.now(), expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index b2768f5..60e8ffe 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -37,6 +37,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { validate: Joi.object().keys({ annoucement: { + role: Joi.string().valid('guest', 'billing').required(), created: Joi.number().required(), expiry: Joi.number().required(), actorKey: KeyVerifier.required(), @@ -190,7 +191,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { let keyDoc = await ctx.party.createDocument('public_key', { created: Date.now(), - role: 'guest', + role: ctx.input.annoucement.role || 'guest', owner: computedActorHash, ...inputActorKey }) diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js new file mode 100644 index 0000000..a178bfc --- /dev/null +++ b/src/venue/bin/commands/venue-remote-check.js @@ -0,0 +1,99 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-check') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + } +} + + +class VenueRemoteCheck extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteCheck.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote check' + } + + static get Definition(){ + return { + usage: `venue remote check`, + description: 'Check a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + console.log('client started') + + this.context.exiting = false + + return {remote, client} + } +} + +module.exports = VenueRemoteCheck + + diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index fc715db..3b4a312 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -21,6 +21,7 @@ commandTree.addCommand(require('./commands/venue-identity-show')) commandTree.addCommand(require('./commands/venue-remote-add')) commandTree.addCommand(require('./commands/venue-remote-list')) commandTree.addCommand(require('./commands/venue-remote-show')) +commandTree.addCommand(require('./commands/venue-remote-check')) commandTree.addCommand(require('./commands/pkg-build')) @@ -118,11 +119,13 @@ async function main(){ await secureConfig.waitForUnlocked('startup') - - const output = await commandTree.run({context: { + + context = { secureConfig, collectPassword, - ...context - }}) + exiting: false + } + + const output = await commandTree.run({context}) if(output){ console.log(output) From 4f2f2c80f7e6f49cd1f5111505f8266341f64ddd Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 15:59:31 +0000 Subject: [PATCH 26/50] improve reconnect timer math --- src/comms/websocket-comms.js | 2 +- src/party/peer/ephemeral-client.js | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index d606cc2..8eb5831 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -14,7 +14,7 @@ const WebsocketShim = require('./websocket-shim') * @see https://en.wikipedia.org/wiki/WebSocket */ class WebsocketComms extends PeerComms { - constructor({uri, connection, timeout=10000, remoteIdentity, host, party, ...options}){ + constructor({uri, connection, timeout=20000, remoteIdentity, host, party, ...options}){ super({remoteIdentity, host, party, ...options}) this.uri = uri diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index c2fd90c..7348eb4 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -9,6 +9,20 @@ const MemoryConfig = require('../../config/memory') const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') +const MAX_RECONNECT_INTERVAL = 120*1000 +const MIN_RECONNECT_INTERVAL = 5*1000 +const MIN_BACKOFF = 3*1000 + +function getReconnectInterval(count, backoff=9000){ + return Math.max( + MIN_RECONNECT_INTERVAL, + Math.min( + MAX_RECONNECT_INTERVAL, + count * Math.max(backoff, MIN_BACKOFF) + ) + ) +} + class EphemeralClient extends EventEmitter { constructor({identity, role='guest', autoreconnect=true, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ @@ -21,6 +35,7 @@ class EphemeralClient extends EventEmitter { this.wsParty = null this.restParty = null this.autoreconnect = autoreconnect + this.backoff = 9000 this.reconnectTimer = null @@ -100,18 +115,24 @@ class EphemeralClient extends EventEmitter { await this.wsParty.start() debug('waiting for websocket authorization') await this.wsParty.comms.authorized() + this.emit('connected') } } async handleWsClose(){ + this.emit('disconnected') + let stopped = this.wsParty.comms.stopped - const sleepTime = Math.min(9000*this.reconnect_tries, 20*1000) + const sleepTime = getReconnectInterval(this.reconnect_tries, this.backoff) debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') if(!this.reconnectTimer){ + + this.emit('reconnecting', {sleepTime, wakeTime: Date.now()+sleepTime}) + this.reconnectTimer = setTimeout( this.doReconnect.bind(this), sleepTime @@ -156,6 +177,9 @@ class EphemeralClient extends EventEmitter { this.reconnect_last_attempt = null this.reconnect_tries = 0 + this.emit('connected') + this.emit('reconnected') + } catch(err){ debug('reconnect error', err) From 9fc0dc2f4377954aea5b8ce3d5e814cd0af49599 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 16:08:53 +0000 Subject: [PATCH 27/50] clean top level signals --- src/party/peer/ephemeral-client.js | 3 ++- src/venue/bin/commands/venue-remote-check.js | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 7348eb4..df9ea1e 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -94,7 +94,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ - + this.emit('connecting') this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -153,6 +153,7 @@ class EphemeralClient extends EventEmitter { this.reconnect_tries++ try{ + this.emit('connecting') this.wsParty.comms = new WebsocketComms({ uri: this.wsUrl, discoverRemoteIdentity: false, diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js index a178bfc..edbf0b1 100644 --- a/src/venue/bin/commands/venue-remote-check.js +++ b/src/venue/bin/commands/venue-remote-check.js @@ -85,6 +85,26 @@ class VenueRemoteCheck extends CmdTree.Command { wsUrlOrParty: remote.ws }) + client.on('connecting',()=>{ + console.log('connecting') + }) + + client.on('connected',()=>{ + console.log('connected') + }) + + client.on('disconnected',()=>{ + console.log('disconnected') + }) + + client.on('reconnected',()=>{ + console.log('reconnected') + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + await client.start() console.log('client started') From 68d480075f112e5519baa6bf5cd9376d9fdd0cd2 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 17:09:09 +0000 Subject: [PATCH 28/50] sane defaults --- src/comms/peer-comms.js | 7 +++ src/comms/websocket-comms.js | 7 +-- src/party/peer/ephemeral-client.js | 47 ++++++++++++++++++-- src/party/peer/peer-party.js | 6 +++ src/venue/bin/commands/venue-remote-check.js | 8 ++++ 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index 65b9438..4402f66 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -55,6 +55,7 @@ class PeerComms extends ISocketComms { this.uuid = uuidv4() this.socket = socket || null this.stopped = false + this.started = false //this.auto_reconnect = !socket && !host this.host = host //! Is comms host\ @@ -205,6 +206,11 @@ class PeerComms extends ISocketComms { async start(){ debug('start') + + if(this.started){ return } + + this.started = true + if(this.socketInit){ await this.socketInit() } @@ -235,6 +241,7 @@ class PeerComms extends ISocketComms { async stop(){ this.stopped = true + this.started = false debug('stop') this.close() } diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index 8eb5831..fbfce40 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -20,6 +20,7 @@ class WebsocketComms extends PeerComms { this.uri = uri this.connection = connection this.timeout = timeout + this.timer = null debug('starting host=',host, ' uuid=', this.uuid, ' uri=', this.uri) @@ -49,7 +50,7 @@ class WebsocketComms extends PeerComms { if(isNewConnection){ //await new Promise((resolve,reject)=>{ - const timer = setTimeout(() => { + this.timer = setTimeout(() => { debug('websocket timeout') this.connection.close() this.emit('timeout') @@ -58,13 +59,13 @@ class WebsocketComms extends PeerComms { this.socket.once('connect', () => { debug('websocket opened') - clearTimeout(timer); + clearTimeout(this.timer); //resolve(); }) this.socket.once('error',(error) => { debug('websocket error', error) - clearTimeout(timer) + clearTimeout(this.timer) this.emit('error', error) //this.connection.close() //reject(error); diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index df9ea1e..0c43c5a 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -13,6 +13,9 @@ const MAX_RECONNECT_INTERVAL = 120*1000 const MIN_RECONNECT_INTERVAL = 5*1000 const MIN_BACKOFF = 3*1000 +const MAX_SESSION_AGE = 24*60*60*1000 //! Set session expiry to 24hr from now +const SESSION_ROLL_AGE = Math.round(MAX_SESSION_AGE * 0.75) + function getReconnectInterval(count, backoff=9000){ return Math.max( MIN_RECONNECT_INTERVAL, @@ -30,6 +33,8 @@ class EphemeralClient extends EventEmitter { this.contacts = contacts this.sessionKey = null + this.sessionExpiry = null + this.sessionTimer = null this.identity = identity this.role = role || 'guest' this.wsParty = null @@ -109,8 +114,6 @@ class EphemeralClient extends EventEmitter { this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) this.wsParty.comms.on('error', this.handleWsClose.bind(this)) - await this.wsParty.start() - debug('starting wsParty') await this.wsParty.start() debug('waiting for websocket authorization') @@ -120,12 +123,40 @@ class EphemeralClient extends EventEmitter { } + async checkSessionExpiry(){ + if(!this.sessionKey){ return } + + const now = Date.now() + + if(this.sessionExpiry <= now){ + await this.rollSessionKey() + } + } + + async rollSessionKey(){ + + debug('rollSessionKey') + this.emit('session-end', this.sessionKey.key.hash) + + if(this.wsParty){ + await this.wsParty.stop() + } + + this.sessionKey = null + this.restParty = null + this.wsParty = null + + await this.start() + } + async handleWsClose(){ this.emit('disconnected') let stopped = this.wsParty.comms.stopped + if(stopped){ return } + const sleepTime = getReconnectInterval(this.reconnect_tries, this.backoff) debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') @@ -193,12 +224,20 @@ class EphemeralClient extends EventEmitter { async announcePublicKeys(callPath='key/announce'){ let currentActor = this.identity + + const now = Date.now() + this.sessionExpiry = now + MAX_SESSION_AGE + this.sessionTimer = setTimeout( + this.rollSessionKey.bind(this), + SESSION_ROLL_AGE + ) + const announceData = { annoucement: { role: this.role, created: Date.now(), - expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + expiry: this.sessionExpiry, sessionKey: { type: this.sessionKey.key.type, hash: this.sessionKey.key.hash, @@ -237,6 +276,8 @@ class EphemeralClient extends EventEmitter { if(announceResult.done != true){ throw new Error('annoucement request failed - '+callPath) } + + this.emit('session', this.sessionKey.key.hash) } diff --git a/src/party/peer/peer-party.js b/src/party/peer/peer-party.js index 889286d..0502d61 100644 --- a/src/party/peer/peer-party.js +++ b/src/party/peer/peer-party.js @@ -62,6 +62,12 @@ class PeerParty extends IParty { await this.comms.start() } + async stop(){ + if(this.comms){ + await this.comms.stop() + } + } + async handleCall(ask){ debug('handleCall') diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js index edbf0b1..37b38aa 100644 --- a/src/venue/bin/commands/venue-remote-check.js +++ b/src/venue/bin/commands/venue-remote-check.js @@ -85,6 +85,14 @@ class VenueRemoteCheck extends CmdTree.Command { wsUrlOrParty: remote.ws }) + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + client.on('connecting',()=>{ console.log('connecting') }) From affd9c891de931a11ea83557a1221b89d24c4ee3 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 17:18:52 +0000 Subject: [PATCH 29/50] randomize reconnect timing --- src/party/peer/ephemeral-client.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 0c43c5a..38283df 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -10,8 +10,8 @@ const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') const MAX_RECONNECT_INTERVAL = 120*1000 -const MIN_RECONNECT_INTERVAL = 5*1000 -const MIN_BACKOFF = 3*1000 +const MIN_RECONNECT_INTERVAL = 9*1000 +const MIN_BACKOFF = 9*1000 const MAX_SESSION_AGE = 24*60*60*1000 //! Set session expiry to 24hr from now const SESSION_ROLL_AGE = Math.round(MAX_SESSION_AGE * 0.75) @@ -21,7 +21,7 @@ function getReconnectInterval(count, backoff=9000){ MIN_RECONNECT_INTERVAL, Math.min( MAX_RECONNECT_INTERVAL, - count * Math.max(backoff, MIN_BACKOFF) + Math.round(MIN_BACKOFF + (count * Math.max(backoff, MIN_BACKOFF) * Math.random())) ) ) } @@ -40,7 +40,7 @@ class EphemeralClient extends EventEmitter { this.wsParty = null this.restParty = null this.autoreconnect = autoreconnect - this.backoff = 9000 + this.backoff = MIN_BACKOFF this.reconnectTimer = null From 90b3d9dba57e74124b9e1688982d06ebf652a37c Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 17:50:46 +0000 Subject: [PATCH 30/50] add timing info --- src/party/peer/ephemeral-client.js | 16 ++++++++-------- src/venue/bin/commands/venue-remote-check.js | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 38283df..6c0c3f5 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -99,7 +99,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ - this.emit('connecting') + this.emit('connecting', {time: Date.now()}) this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -118,7 +118,7 @@ class EphemeralClient extends EventEmitter { await this.wsParty.start() debug('waiting for websocket authorization') await this.wsParty.comms.authorized() - this.emit('connected') + this.emit('connected', {time: Date.now()}) } } @@ -136,7 +136,7 @@ class EphemeralClient extends EventEmitter { async rollSessionKey(){ debug('rollSessionKey') - this.emit('session-end', this.sessionKey.key.hash) + this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) if(this.wsParty){ await this.wsParty.stop() @@ -151,7 +151,7 @@ class EphemeralClient extends EventEmitter { async handleWsClose(){ - this.emit('disconnected') + this.emit('disconnected', {time: Date.now()}) let stopped = this.wsParty.comms.stopped @@ -184,7 +184,7 @@ class EphemeralClient extends EventEmitter { this.reconnect_tries++ try{ - this.emit('connecting') + this.emit('connecting', {time:Date.now()}) this.wsParty.comms = new WebsocketComms({ uri: this.wsUrl, discoverRemoteIdentity: false, @@ -209,8 +209,8 @@ class EphemeralClient extends EventEmitter { this.reconnect_last_attempt = null this.reconnect_tries = 0 - this.emit('connected') - this.emit('reconnected') + this.emit('connected', {time:Date.now()}) + this.emit('reconnected', {time:Date.now()}) } catch(err){ debug('reconnect error', err) @@ -277,7 +277,7 @@ class EphemeralClient extends EventEmitter { throw new Error('annoucement request failed - '+callPath) } - this.emit('session', this.sessionKey.key.hash) + this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) } diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js index 37b38aa..bae2595 100644 --- a/src/venue/bin/commands/venue-remote-check.js +++ b/src/venue/bin/commands/venue-remote-check.js @@ -93,20 +93,20 @@ class VenueRemoteCheck extends CmdTree.Command { console.log('session-end', id) }) - client.on('connecting',()=>{ - console.log('connecting') + client.on('connecting',(info)=>{ + console.log('connecting', info) }) - client.on('connected',()=>{ - console.log('connected') + client.on('connected',(info)=>{ + console.log('connected', info) }) - client.on('disconnected',()=>{ - console.log('disconnected') + client.on('disconnected',(info)=>{ + console.log('disconnected', info) }) - client.on('reconnected',()=>{ - console.log('reconnected') + client.on('reconnected',(info)=>{ + console.log('reconnected', info) }) client.on('reconnecting',(info)=>{ From 68f5446508bcb2bad23a6d11a4e29e214caea0e6 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 20:39:23 +0000 Subject: [PATCH 31/50] project build --- src/venue/bin/commands/project-build.js | 127 ++++++++++++++++++++++-- src/venue/bin/venue.js | 1 + src/venue/example-project.js | 25 +++++ 3 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 src/venue/example-project.js diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 038f218..0b2aad6 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -10,13 +10,19 @@ const prompt = require('prompt') const argon2 = require('argon2') const { execSync } = require('child_process') -const findUp = require('find-up-json').default + +const { + globSync +} = require('glob') +const tar = require('tar') const Dataparty = require('../../../../') const dataparty_crypto = require('@dataparty/crypto') const Joi = require('joi') +const {Routines} = dataparty_crypto + const DEFINITION = { h: { description: 'Show help', @@ -50,10 +56,60 @@ const DEFINITION = { } +async function compressFiles(projectName, root, fileList, outputPath, writeFile){ + + if(!fileList){ return } + + let fileMap={} + + let files = fileList.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + if(!files || files.length < 1){ return } + + const tarFileName = projectName.replace('/', '-')+'.project.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) + + await tar.create({ + cwd: root, + gzip: true, + file: tarPath + }, fileList) + + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + const fileInfo = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } + + return {tarPath, files: fileInfo} +} + + class VenueProjectBuild extends CmdTree.Command { constructor(context){ super({...VenueProjectBuild.Definition, context}) debug('constructor') + + this.project = {} + this.project_sources = [] } static get Command(){ @@ -77,7 +133,7 @@ class VenueProjectBuild extends CmdTree.Command { } if (parsed._.length != 3){ - throw new CmdTree.Error.UsageError('You must supply project json') + throw new CmdTree.Error.UsageError('You must supply project json/js') } const keyName = parsed.identity @@ -102,9 +158,9 @@ class VenueProjectBuild extends CmdTree.Command { const projectJsonPath = Path.resolve(parsed._[2]) - let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + /*let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) - let pkgJson = foundUp.content + let pkgJson = foundUp.content*/ let projectJson = require( projectJsonPath ) @@ -117,22 +173,75 @@ class VenueProjectBuild extends CmdTree.Command { } const project = { + owner: key.key.hash, + created: Date.now(), + name: parsed.name ? parsed.name : projectJson.name, version: parsed.version ? parsed.version : projectJson.version, - owner: key.id, - created: Date.now(), venue: remote.identity.key.hash, domain: projectJson.domain, + i2p: projectJson.i2p, + party: projectJson.party, + routes: projectJson.routes, + files: projectJson.files + } await mkdirp(parsed.output) - const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(parsed.output, true, key) + const buildOutput = parsed.output+'/'+ project.name.replace('/', '-') +'.project.venue.json' + + let prjFiles = [] + prjFiles.push(buildOutput) + + if(project.files){ + this.addProjectFiles( + Path.dirname(projectJsonPath), + projectJson.files, + { nodir: true, follow: true } + ) + + const {tarPath, files} = await compressFiles(project.name,Path.dirname(projectJsonPath), this.project_sources.files, parsed.output, true) + + project.files = files + + if(tarPath){prjFiles.push(tarPath)} + + + } + + const ownerSig = await key.sign( project, true ) + + project.signatures = { + [key.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + + fs.writeFileSync(buildOutput, JSON.stringify(project, null,2)) + + return {files: prjFiles, project} + } + + + addProjectFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.project_sources.files){ + this.project_sources.files = result + } else { + this.project_sources.files = this.project_sources.files.concat(result) + } + + this.project_sources.files_root = root + + debug('addFiles',result) - return {files: build.files} } } diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index 3b4a312..b405c7a 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -24,6 +24,7 @@ commandTree.addCommand(require('./commands/venue-remote-show')) commandTree.addCommand(require('./commands/venue-remote-check')) commandTree.addCommand(require('./commands/pkg-build')) +commandTree.addCommand(require('./commands/project-build')) const HOMEDIR = OS.homedir() const DEFAULT_FOLDER = '.venue' diff --git a/src/venue/example-project.js b/src/venue/example-project.js new file mode 100644 index 0000000..44b93e9 --- /dev/null +++ b/src/venue/example-project.js @@ -0,0 +1,25 @@ +module.exports = { + owner: 'null', + name: 'example-project', + version: '1.0', + + venue: 'dataparty-venue', + domain: 'api.dataparty.xyz', + + routes: [{ + prefix: '/api', + party:'SYSTEM', + package: { + name: '@dataparty/venue' + }, + settings: { + sendFullErrors: false, + useNative: false + } + }], + files: [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ] + } \ No newline at end of file From f3d0e3be025f62637a92dae91026727f548245f1 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 21:38:00 +0000 Subject: [PATCH 32/50] suport deploy --- src/venue/bin/commands/pkg-build.js | 42 +++++++++++++++++++++++++ src/venue/bin/commands/project-build.js | 32 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js index 11a9db7..5bcaca5 100644 --- a/src/venue/bin/commands/pkg-build.js +++ b/src/venue/bin/commands/pkg-build.js @@ -41,6 +41,14 @@ const DEFINITION = { }, version: { description: 'package version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false } } @@ -113,8 +121,42 @@ class VenuePackageBuild extends CmdTree.Command { const builder = new Dataparty.ServiceBuilder(service) const build = await builder.compile(parsed.output, true, key) + + if(parsed.deploy && parsed.remote){ + console.log('uploading...') + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + let staticTar = undefined + + if(build.files.length == 3){ + staticTar = fs.readFileSync(build.files[ build.files.length - 1 ]) + } + + await this.pushPackage(key, remote, build.build, staticTar) + } + return {files: build.files} } + + + async pushPackage(devId, remote, build, staticTar){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } } module.exports = VenuePackageBuild diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 0b2aad6..3708280 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -52,6 +52,10 @@ const DEFINITION = { remote: { type: 'string', description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false } } @@ -220,6 +224,14 @@ class VenueProjectBuild extends CmdTree.Command { fs.writeFileSync(buildOutput, JSON.stringify(project, null,2)) + let staticTar = undefined + + if(prjFiles.length == 3){ + staticTar = fs.readFileSync(prjFiles[ prjFiles.length - 1 ]) + } + + await this.pushProject(key, remote, project, staticTar) + return {files: prjFiles, project} } @@ -243,6 +255,26 @@ class VenueProjectBuild extends CmdTree.Command { debug('addFiles',result) } + + async pushProject(devId, remote, build, staticTar){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-project', {project:build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } } module.exports = VenueProjectBuild From a393621afbf7464e5d56b81209d503a867ce52a2 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Tue, 16 Jun 2026 22:12:55 +0000 Subject: [PATCH 33/50] clients --- src/party/peer/ephemeral-client.js | 5 + src/party/peer/match-maker-client.js | 283 +++++---------------------- src/party/peer/peer-client.js | 64 ++++++ src/party/peer/peer-invite.js | 41 +++- 4 files changed, 157 insertions(+), 236 deletions(-) create mode 100644 src/party/peer/peer-client.js diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 6c0c3f5..04321c5 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -65,6 +65,7 @@ class EphemeralClient extends EventEmitter { async start(){ + if(this.sessionKey){ return } this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) if(!this.restParty){ @@ -123,6 +124,10 @@ class EphemeralClient extends EventEmitter { } + get socketPeerParty(){ + return this.wsParty + } + async checkSessionExpiry(){ if(!this.sessionKey){ return } diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 7c9b9ff..cc4dee7 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -12,30 +12,11 @@ const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws', billingIdentity=null){ + constructor(client){ super() - - this.contacts = contacts - this.sessionKey = null - this.identity = identity - this.wsParty = null - this.restParty = null - this.billingIdentity = null - - if(typeof urlOrParty == 'string'){ - this.restUrl = urlOrParty - this.restParty = null - } else { - this.restParty = urlOrParty - } - if(typeof wsUrlOrParty == 'string'){ - this.wsUrl = wsUrlOrParty - this.wsParty = null - } else { - this.wsParty = wsUrlOrParty - } + this.client = client this.invitesTx = null this.invitesRx = null @@ -44,80 +25,55 @@ class MatchMakerClient extends EventEmitter { tx: {}, rx: {} } + + this.started = false + + this.client.on('connected', this.handleConnect.bind(this)) + this.client.on('disconnected', this.handleDisconnect.bind(this)) } + async handleConnect(){ + if(!this.started){ return } + debug('handleConnect') + await this.start() + } - async start(){ - this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) - - if(!this.restParty){ - let config = new MemoryConfig({ - basePath:'match-maker-client', - cloud: { - uri: this.restUrl - } - }) - - this.restParty = new LokiParty({ - path: 'match-maker-client', - dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), - config - }) - - await this.restParty.setIdentity(this.sessionKey) - - debug('starting restParty') - await this.restParty.start() - - if(!this.restParty.comms){ - this.restParty.comms = new RestComms({ - party:this.restParty, - config: this.restParty.config - }) - - this.restParty.comms.sessionId = this.sessionKey.key.hash - } + async handleDisconnect(){ + if(!this.started){ return } - await this.announcePublicKeys() - } + debug('handleDisconnect') + this.invitesRx.unsubscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesTx.unsubscribe( this.handleInviteTxMsg.bind(this) ) - if(!this.wsParty && this.wsUrl){ - this.wsParty = new PeerParty({ - comms: new WebsocketComms({ - uri: this.wsUrl, - discoverRemoteIdentity: false, - remoteIdentity: await this.restParty.comms.getServiceIdentity(), - session: this.sessionKey.key.hash - }), - config: this.restParty.config - }) + this.emit('disconnected') + } - await this.wsParty.start() + async start(){ - debug('starting wsParty') - await this.wsParty.start() - debug('waiting for websocket authorization') - await this.wsParty.comms.authorized() + this.started = true + await this.client.start() - this.invitesRx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/rx', - messageType: 'Object' - }) + const party = this.client.socketPeerParty - this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesRx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/rx', + messageType: 'Object' + }) + this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) - this.invitesTx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/tx', - messageType: 'Object' - }) - this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) - } - + this.invitesTx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/tx', + messageType: 'Object' + }) + + this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) + + this.emit('connected') } async handleInviteRxMsg( msg ){ @@ -127,8 +83,8 @@ class MatchMakerClient extends EventEmitter { if(!this.pendingInvites.rx[inviteId] && msg.invite.state == 'invited'){ - const from = await this.lookupPublicKey(msg.invite.fromHash) - const to = await this.lookupPublicKey(msg.invite.toHash) + const from = await this.client.lookupPublicKey(msg.invite.fromHash) + const to = await this.client.lookupPublicKey(msg.invite.toHash) let invite = new PeerInvite(msg.invite, to, this, from) @@ -161,95 +117,6 @@ class MatchMakerClient extends EventEmitter { } } - async announceBillingKey({stripeCheckoutSession}={}){ - this.announcePublicKeys(true, { - stripe: stripeCheckoutSession - }) - } - - async announcePublicKeys(useBillingKeyAsActor=false, billingMethodDetails=null){ - - let currentActor = useBillingKeyAsActor == true ? this.billingIdentity : this.identity - - const announceData = { - annoucement: { - //type: 'guest',//useBillingKeyAsActor ? 'billing_identity' : 'user_identity', - created: Date.now(), - expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now - sessionKey: { - type: this.sessionKey.key.type, - hash: this.sessionKey.key.hash, - public: this.sessionKey.key.public - }, - actorKey: { - type: currentActor.key.type, - hash: currentActor.key.hash, - public: currentActor.key.public - } - }, - trust: { - actorSig: null, - sessionSig: null - } - } - - - const actorSigMsg = await currentActor.sign(announceData.annoucement, true) - const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) - - debug('actorSigMsg', actorSigMsg) - debug('sessionSigMsg', sessionSigMsg) - - announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) - announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) - - debug('announcePublicKeys', announceData) - - let callPath = useBillingKeyAsActor ? 'billing/key/announce' : 'key/announce' - - const announceResult = await this.restParty.comms.call(callPath, announceData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: false - }) - - if(announceResult.done != true){ - throw new Error('annoucement request failed - '+callPath) - } - } - - - async lookupPublicKey(hash){ - debug('lookupPublicKey - hash:', hash) - - if(hash == this.identity.key.hash){ - return this.identity - } - - if(this.contacts){ - return await this.contacts.lookupPublicKey(hash) - } - - const lookupData = { hash } - - const lookupResult = await this.restParty.comms.call('key/lookup', lookupData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - if(!lookupResult.done){ - return null - } - - debug('lookup result -', lookupResult) - - const identity = new dataparty_crypto.Identity({ - key: lookupResult.public_key - }) - - return identity - } async createInvite(toHashOrIdentity, {type, service, role, session}, info){ @@ -257,7 +124,7 @@ class MatchMakerClient extends EventEmitter { let toIdentity = null if(typeof toHashOrIdentity == 'string'){ - toIdentity = await this.lookupPublicKey(toHashOrIdentity) + toIdentity = await this.client.lookupPublicKey(toHashOrIdentity) } else { toIdentity = toHashOrIdentity } @@ -269,7 +136,7 @@ class MatchMakerClient extends EventEmitter { service: service ? service : '@dataparty/video-chat', role: role ? role : 'client', timestamp: (new Date()).getTime(), - from: this.identity.key.hash, + from: this.client.identity.key.hash, to: toIdentity.key.hash, session: session ? session : Math.random().toString(36).slice(2), info: info ? info : { @@ -278,17 +145,17 @@ class MatchMakerClient extends EventEmitter { } } - const secureInvite = await this.identity.encrypt(invitePayload, toIdentity) + const secureInvite = await this.client.identity.encrypt(invitePayload, toIdentity) debug('secure-invite', secureInvite) const invitePostData = { to: toIdentity.key.hash, - from: this.identity.key.hash, + from: this.client.identity.key.hash, payload: JSON.stringify(secureInvite.toJSON()) } - const inviteResult = await this.restParty.comms.call('invite/create', invitePostData, { + const inviteResult = await this.client.socketPeerParty.comms.call('invite/create', invitePostData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -298,9 +165,9 @@ class MatchMakerClient extends EventEmitter { if(!inviteDoc){ return } - let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.identity) + let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.client.identity, invitePayload) - invite.payload = invitePayload + //invite.payload = invitePayload this.pendingInvites.tx[inviteDoc.$meta.id] = invite @@ -310,16 +177,16 @@ class MatchMakerClient extends EventEmitter { } async lookupInvites({createdAfter, type='to', id, actorHash }){ - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const lookup = { invite: id, - actor: actorHash ? actorHash : this.identity.key.hash, + actor: actorHash ? actorHash : this.client.identity.key.hash, createdAfter, type: !type ? 'to' : type } - const lookupResult = await this.restParty.comms.call('invite/lookup', lookup, { + const lookupResult = await this.client.socketPeerParty.comms.call('invite/lookup', lookup, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -355,8 +222,8 @@ class MatchMakerClient extends EventEmitter { for(let i=0; i < invites.length; i++){ const invite = invites[i] - let to = await this.lookupPublicKey( invite.toHash ) - let from = await this.lookupPublicKey( invite.fromHash ) + let to = await this.client.lookupPublicKey( invite.toHash ) + let from = await this.client.lookupPublicKey( invite.fromHash ) let peerInvite = new PeerInvite( invites[i], to, this, from) @@ -377,14 +244,14 @@ class MatchMakerClient extends EventEmitter { async setInviteState(invite, newState){ debug('setInviteState') - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const inviteState = { invite: invite.inviteDoc.$meta.id, state: newState } - const inviteStateResult = await this.restParty.comms.call('invite/set-state', inviteState, { + const inviteStateResult = await this.client.socketPeerParty.comms.call('invite/set-state', inviteState, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -399,48 +266,6 @@ class MatchMakerClient extends EventEmitter { return inviteStateResult.invite } - async createShortCode(use_limit=3, expiry){ - debug('createShortCode') - - const request = { - use_limit, - expiry: !expiry ? Date.now()+24*60*60*3 : expiry - } - - const result = await this.restParty.comms.call('short-code/create', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('createShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } - - async lookupPublicKeyByShortCode( code ){ - debug('lookupPublicKeyByShortCode') - - const request = { code } - - const result = await this.restParty.comms.call('short-code/lookup', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('lookupPublicKeyByShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } } module.exports = MatchMakerClient diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js new file mode 100644 index 0000000..d614126 --- /dev/null +++ b/src/party/peer/peer-client.js @@ -0,0 +1,64 @@ +const EphemeralClient = require("./ephemeral-client") + + +class PeerClient extends EphemeralClient { + constructor({config, hostParty, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + + this.config = config + this.hostParty = hostParty + this.matchMaker = matchMaker + + this.remoteIdentityHash = remoteIdentityHash + + this.inviteSettings = { + type: 'webrtc', + service: service, + role: role ? role : 'client', + session: null + } + + this.rtcSettings = this.rtcSettings + + this.peerParty = null + } + + async start(mediaSrc){ + // + + if(this.sessionKey){ return } + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + this.inviteSettings.session = this.sessionKey.key.hash + + this.emit('connecting', {time: Date.now()}) + let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings) + + await invite.waitForAccepted() + + this.peerParty = await invite.establish({ + mediaSrc, + hostParty: this.hostParty, + config: this.config, + rtcSettings: this.rtcSettings + }) + + this.emit('connected', {time: Date.now()}) + + return this.peerParty + } + + async rollSessionKey(){ + // + } + + async handleClose(){ + // + } + + async doReconnect(){ + // + } + +} + +module.exports = PeerClient diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index dbd3f8b..f2cb05b 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -8,6 +8,8 @@ const dataparty_crypto = require('@dataparty/crypto') const PeerParty = require('./peer-party') const RTCSocketComms = require('../../comms/rtc-socket-comms') +const DEFAULT_EXPIRY = 5*60*1000 + const END_STATES = [ 'cancelled', 'rejected', 'expired', 'completed' ] @@ -33,7 +35,7 @@ async function delay(ms){ } class PeerInvite extends EventEmitter { - constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity){ + constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity, payload=null){ super() this.peerParty = null @@ -42,7 +44,7 @@ class PeerInvite extends EventEmitter { this.matchMaker = matchMakerClient this.inviteDoc = inviteDoc this.inviteMsg = null //this.latestDoc = null - this.payload = null + this.payload = payload this.topicSub = null this.topicPub = null @@ -54,6 +56,18 @@ class PeerInvite extends EventEmitter { this.incomingStream = null + this.timeoutTimer = null + + if(this.payload){ + const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() + + const delta = expiry - now + if(delta > 0){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + } + } + /*if(!this.isSender()){ this.inviteDoc. }*/ @@ -89,7 +103,7 @@ class PeerInvite extends EventEmitter { async accept(mediaSrc, config){ debug('accepting invite') - /*if(this.inviteDoc.toHash == matchMaker.wsParty.identity.key.hash){ + /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.fromHash) } else { otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.toHash) @@ -102,12 +116,20 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(JSON.parse(changedInvite.payload)) - let payload = await this.matchMaker.identity.decrypt( + let payload = await this.matchMaker.client.identity.decrypt( msgWorkAround ) this.payload = payload.msg + /*const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() + + const delta = expiry - now + if(delta > 0 && this.timeoutTimer != null){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + }*/ + return await this.establish({mediaSrc, config}) } @@ -120,6 +142,10 @@ class PeerInvite extends EventEmitter { return (this.inviteMsg || this.inviteDoc).state } + async handleTimeout(){ + await this.matchMaker.setInviteState(this, 'expired') + } + async onInviteMsg(inviteMsg){ debug('onInviteMsg', inviteMsg) @@ -168,7 +194,7 @@ class PeerInvite extends EventEmitter { let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from - let party = this.matchMaker.wsParty + let party = this.this.matchMaker.client.socketPeerParty this.topicSub = new party.ROSLIB.Topic({ ros : party.comms.ros, @@ -190,7 +216,7 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(msg.offers[i]) - let offer = await this.matchMaker.identity.decrypt(msgWorkAround) + let offer = await this.matchMaker.client.identity.decrypt(msgWorkAround) if(offer.from.hash != otherIdentity.key.hash){ debug('BAD IDENTITY') @@ -268,7 +294,7 @@ class PeerInvite extends EventEmitter { debug(' >> offer signal trickle', data) - const secureOffer = await this.matchMaker.identity.encrypt(data, otherIdentity) + const secureOffer = await this.matchMaker.client.identity.encrypt(data, otherIdentity) if(host && !sendFreely){ //console.log('am host') @@ -312,6 +338,7 @@ class PeerInvite extends EventEmitter { } catch (err){ console.log(err) + throw err } } From 270c5e859c20e6efd4174b51f246099a57a94898 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 19 Jun 2026 20:35:26 +0000 Subject: [PATCH 34/50] cleanup duplicate code --- src/party/peer/ephemeral-client.js | 154 ++---------------- src/party/peer/match-maker-client.js | 15 +- src/party/peer/peer-client.js | 24 ++- src/party/peer/peer-invite.js | 4 +- src/service/index-browser.js | 8 + src/service/index.js | 11 +- src/venue/tasks/cleanup-ephemeral-sessions.js | 86 ---------- src/venue/venue-service.js | 2 +- 8 files changed, 64 insertions(+), 240 deletions(-) delete mode 100644 src/venue/tasks/cleanup-ephemeral-sessions.js diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 04321c5..ce36b7c 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -9,40 +9,17 @@ const MemoryConfig = require('../../config/memory') const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') -const MAX_RECONNECT_INTERVAL = 120*1000 -const MIN_RECONNECT_INTERVAL = 9*1000 -const MIN_BACKOFF = 9*1000 - -const MAX_SESSION_AGE = 24*60*60*1000 //! Set session expiry to 24hr from now -const SESSION_ROLL_AGE = Math.round(MAX_SESSION_AGE * 0.75) - -function getReconnectInterval(count, backoff=9000){ - return Math.max( - MIN_RECONNECT_INTERVAL, - Math.min( - MAX_RECONNECT_INTERVAL, - Math.round(MIN_BACKOFF + (count * Math.max(backoff, MIN_BACKOFF) * Math.random())) - ) - ) -} - class EphemeralClient extends EventEmitter { - constructor({identity, role='guest', autoreconnect=true, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + constructor({identity, role='guest', contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ super() this.contacts = contacts this.sessionKey = null - this.sessionExpiry = null - this.sessionTimer = null this.identity = identity this.role = role || 'guest' this.wsParty = null this.restParty = null - this.autoreconnect = autoreconnect - this.backoff = MIN_BACKOFF - - this.reconnectTimer = null if(typeof urlOrParty == 'string'){ this.restUrl = urlOrParty @@ -57,15 +34,11 @@ class EphemeralClient extends EventEmitter { } else { this.wsParty = wsUrlOrParty } - - this.reconnect_last_attempt = null - this.reconnect_tries = 0 } async start(){ - if(this.sessionKey){ return } this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) if(!this.restParty){ @@ -100,7 +73,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ - this.emit('connecting', {time: Date.now()}) + this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -111,138 +84,33 @@ class EphemeralClient extends EventEmitter { config: this.restParty.config }) - this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) - this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) - this.wsParty.comms.on('error', this.handleWsClose.bind(this)) - - debug('starting wsParty') - await this.wsParty.start() - debug('waiting for websocket authorization') - await this.wsParty.comms.authorized() - this.emit('connected', {time: Date.now()}) - } - - } - - get socketPeerParty(){ - return this.wsParty - } + this.wsParty.comms.on('close', ()=>{ - async checkSessionExpiry(){ - if(!this.sessionKey){ return } - - const now = Date.now() - - if(this.sessionExpiry <= now){ - await this.rollSessionKey() - } - } - - async rollSessionKey(){ - - debug('rollSessionKey') - this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) - - if(this.wsParty){ - await this.wsParty.stop() - } - - this.sessionKey = null - this.restParty = null - this.wsParty = null - - await this.start() - } + let stopped = this.wsParty.comms.stopped + console.log('hey the ws closed - stopped=', stopped) - async handleWsClose(){ - - this.emit('disconnected', {time: Date.now()}) - - let stopped = this.wsParty.comms.stopped - - if(stopped){ return } - - const sleepTime = getReconnectInterval(this.reconnect_tries, this.backoff) - debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') - - if(!this.reconnectTimer){ - - this.emit('reconnecting', {sleepTime, wakeTime: Date.now()+sleepTime}) - - this.reconnectTimer = setTimeout( - this.doReconnect.bind(this), - sleepTime - ) - } - } - - async doReconnect(){ - let stopped = this.wsParty.comms.stopped - - this.reconnectTimer = null - - if(stopped || !this.wsParty || !this.autoreconnect){ debug('skip reconnect'); return } - - debug('doing ws reconnect...') - - this.reconnect_last_attempt = Date.now() - this.reconnect_tries++ - - try{ - this.emit('connecting', {time:Date.now()}) - this.wsParty.comms = new WebsocketComms({ - uri: this.wsUrl, - discoverRemoteIdentity: false, - remoteIdentity: await this.restParty.comms.getServiceIdentity(), - session: this.sessionKey.key.hash }) - this.wsParty.comms.party = this.wsParty - - this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) - this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) - this.wsParty.comms.on('error', this.handleWsClose.bind(this)) + await this.wsParty.start() - debug('restarting websocket') - await this.wsParty.comms.start() + debug('starting wsParty') + await this.wsParty.start() debug('waiting for websocket authorization') await this.wsParty.comms.authorized() - - - debug('connected and authorized') - - this.reconnect_last_attempt = null - this.reconnect_tries = 0 - - this.emit('connected', {time:Date.now()}) - this.emit('reconnected', {time:Date.now()}) - - } catch(err){ - debug('reconnect error', err) - - - this.handleWsClose() } + } async announcePublicKeys(callPath='key/announce'){ let currentActor = this.identity - - const now = Date.now() - this.sessionExpiry = now + MAX_SESSION_AGE - this.sessionTimer = setTimeout( - this.rollSessionKey.bind(this), - SESSION_ROLL_AGE - ) - const announceData = { annoucement: { role: this.role, created: Date.now(), - expiry: this.sessionExpiry, + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { type: this.sessionKey.key.type, hash: this.sessionKey.key.hash, @@ -281,8 +149,6 @@ class EphemeralClient extends EventEmitter { if(announceResult.done != true){ throw new Error('annoucement request failed - '+callPath) } - - this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) } diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index cc4dee7..14dbb99 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -28,29 +28,36 @@ class MatchMakerClient extends EventEmitter { this.started = false - this.client.on('connected', this.handleConnect.bind(this)) + this.client.on('reconnected', this.handleReconnect.bind(this)) this.client.on('disconnected', this.handleDisconnect.bind(this)) } - async handleConnect(){ - if(!this.started){ return } + async handleReconnect(){ + //if(!this.started){ return } - debug('handleConnect') + debug('handleReconnect') await this.start() } async handleDisconnect(){ if(!this.started){ return } + thi.started = false + debug('handleDisconnect') this.invitesRx.unsubscribe( this.handleInviteRxMsg.bind(this) ) this.invitesTx.unsubscribe( this.handleInviteTxMsg.bind(this) ) + this.invitesRx = null + this.invitesTx = null + this.emit('disconnected') } async start(){ + if(this.started){ return } + this.started = true await this.client.start() diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index d614126..4292ceb 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -31,7 +31,7 @@ class PeerClient extends EphemeralClient { this.inviteSettings.session = this.sessionKey.key.hash this.emit('connecting', {time: Date.now()}) - let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings) + const invite = await this.announcePublicKeys() await invite.waitForAccepted() @@ -48,7 +48,17 @@ class PeerClient extends EphemeralClient { } async rollSessionKey(){ - // + debug('rollSessionKey') + this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) + + if(this.peerParty){ + await this.peerParty.stop() + } + + this.sessionKey = null + this.peerParty = null + + await this.start() } async handleClose(){ @@ -58,6 +68,16 @@ class PeerClient extends EphemeralClient { async doReconnect(){ // } + + async announcePublicKeys(){ + const announceData = await this.createSessionAnnoucement() + + let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings, announceData) + + this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) + + return invite + } } diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index f2cb05b..eeb7ec0 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -100,7 +100,7 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept(mediaSrc, config){ + async accept(mediaSrc, config, hostParty){ debug('accepting invite') /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ @@ -130,7 +130,7 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) }*/ - return await this.establish({mediaSrc, config}) + return await this.establish({mediaSrc, config, hostParty}) } async reject(){ diff --git a/src/service/index-browser.js b/src/service/index-browser.js index e026bf2..d33a83a 100644 --- a/src/service/index-browser.js +++ b/src/service/index-browser.js @@ -50,4 +50,12 @@ exports.endpoint_paths = { secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), version: Path.join(__dirname, './endpoints/service-version.js') +} + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/service/index.js b/src/service/index.js index 9f00a91..0472ce0 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -50,7 +50,7 @@ exports.endpoint = { secureecho: require('./endpoints/secure-echo'), identity: require('./endpoints/service-identity'), version: require('./endpoints/service-version'), - key_announce: require('./endpoints/key-announce'), + key_announce: require('./endpoints/key-announce') } exports.endpoint_paths = { @@ -69,4 +69,13 @@ exports.schema = { exports.schema_paths = { public_key: Path.join(__dirname, './schema/public-key.js'), session_key: Path.join(__dirname, './schema/session-key.js') +} + + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/venue/tasks/cleanup-ephemeral-sessions.js b/src/venue/tasks/cleanup-ephemeral-sessions.js deleted file mode 100644 index 435a48b..0000000 --- a/src/venue/tasks/cleanup-ephemeral-sessions.js +++ /dev/null @@ -1,86 +0,0 @@ -const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') - -//const ITask = require('@dataparty/api/src/service/itask') -const ITask = require('../../service/itask') - -class CleanupEphemeralSessionsTask extends ITask { - - constructor(options){ - super({ - name: CleanupEphemeralSessionsTask.name, - background: CleanupEphemeralSessionsTask.Config.background, - ...options - }) - - debug('new') - - this.duration = Math.round(1000*60*15) - this.timeout = null - } - - static get Config(){ - return { - background: true, - autostart: true - } - } - - async exec(){ - - this.setTimer() - - return this.detach() - } - - - async lookupSessions(){ - - let now = Date.now() - - return (await this.context.party.find() - .type('session_key') - .where('expiry').lt(now) - .exec()) - } - - setTimer(){ - this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) - } - - async onTimeout(){ - this.timeout = null - - debug('cleanup ephemeral sessions task') - - try{ - let sessions = await this.lookupSessions() - - if(sessions && sessions.length > 0){ - debug('expired sessions ', sessions.length) - let list = sessions.map(i=>{return i.data}) - await this.context.party.remove(...list) - } - } catch (err){ - debug(err) - } - - this.setTimer() - } - - stop(){ - if(this.timeout !== null){ - clearTimeout(this.timeout) - this.timeout = null - } - } - - static get Name(){ - return 'cleanup-ephemeral-sessions' - } - - static get Description(){ - return 'Cleanup Ephemeral sessions' - } -} - -module.exports = CleanupEphemeralSessionsTask \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index 815ea87..444c358 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -39,7 +39,7 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) - builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) + builder.addTask(DatapartySrv.task_paths.cleanup_ephemeral_sessions) builder.addAuth(Path.join(__dirname, './auth.js')) From 3ec1a77e7a22417d8361df3d2bcf338aeb098d12 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 19 Jun 2026 20:40:27 +0000 Subject: [PATCH 35/50] missed cleanup task move and add start of peer client --- src/service/service-host-peer.js | 32 +++++++ .../tasks/cleanup-ephemeral-sessions.js | 86 +++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 src/service/service-host-peer.js create mode 100644 src/service/tasks/cleanup-ephemeral-sessions.js diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js new file mode 100644 index 0000000..bebaded --- /dev/null +++ b/src/service/service-host-peer.js @@ -0,0 +1,32 @@ +const debug = require('debug')('dataparty.service.host-peer') + +class ServiceHostPeer { + + constructor({ + runner, + matchMaker, + mediaSrc + + }){ + this.runner = runner + this.matchMaker = matchMaker + + this.mediaSrc = mediaSrc + } + + async start(){ + // + + this.matchMaker.on('invited', this.onInvite.bind(this)) + } + + async onInvite(invite){ + // + + //! check if service wants to allow user + + const peerParty = await invite.accept(this.mediaSrc,this.config) + } +} + +module.exports = ServiceHostPeer \ No newline at end of file diff --git a/src/service/tasks/cleanup-ephemeral-sessions.js b/src/service/tasks/cleanup-ephemeral-sessions.js new file mode 100644 index 0000000..18cb7d0 --- /dev/null +++ b/src/service/tasks/cleanup-ephemeral-sessions.js @@ -0,0 +1,86 @@ +const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') + +//const ITask = require('@dataparty/api/src/service/itask') +const ITask = require('../../service/itask') + +class CleanupEphemeralSessionsTask extends ITask { + + constructor(options){ + super({ + name: CleanupEphemeralSessionsTask.name, + background: CleanupEphemeralSessionsTask.Config.background, + ...options + }) + + debug('new') + + this.duration = Math.round(1000*60*15) + this.timeout = null + } + + static get Config(){ + return { + background: true, + autostart: true + } + } + + async exec(){ + + this.setTimer() + + return this.detach() + } + + + async lookupSessions(){ + + let now = Date.now() + + return (await this.context.party.find() + .type('session_key') + .where('expiry').lt(now) + .exec()) + } + + setTimer(){ + this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) + } + + async onTimeout(){ + this.timeout = null + + debug('cleanup ephemeral sessions task') + + try{ + let sessions = await this.lookupSessions() + + if(sessions && sessions.length > 0){ + debug('expired sessions ', sessions.length) + let list = sessions.map(i=>{return i.data}) + await this.context.party.remove(...list) + } + } catch (err){ + debug(err) + } + + this.setTimer() + } + + stop(){ + if(this.timeout !== null){ + clearTimeout(this.timeout) + this.timeout = null + } + } + + static get Name(){ + return 'cleanup-ephemeral-sessions' + } + + static get Description(){ + return 'Cleanup Ephemeral sessions' + } +} + +module.exports = CleanupEphemeralSessionsTask From 8151261fb494e94041a6215fdd22229c789a5b6f Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 16:44:13 +0000 Subject: [PATCH 36/50] improve role logic and access in PeerInvite --- src/party/peer/match-maker-client.js | 6 +++++ src/party/peer/peer-client.js | 7 ++++-- src/party/peer/peer-invite.js | 32 ++++++++++++++++++++------ src/service/runner-router.js | 4 ++-- src/service/service-host-peer.js | 34 +++++++++++++++++++++++++--- src/venue/venue-service.js | 5 ---- 6 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 14dbb99..7847d4e 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -127,6 +127,12 @@ class MatchMakerClient extends EventEmitter { async createInvite(toHashOrIdentity, {type, service, role, session}, info){ + const roles = ['client', 'host'] + + if(roles.indexOf(role) == -1){ + throw new Error("Invalid requested role [" + role + "]") + } + debug('createInvite') let toIdentity = null diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index 4292ceb..c14bc18 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -30,6 +30,8 @@ class PeerClient extends EphemeralClient { this.inviteSettings.session = this.sessionKey.key.hash + const role = this.inviteSettings.role + this.emit('connecting', {time: Date.now()}) const invite = await this.announcePublicKeys() @@ -37,8 +39,9 @@ class PeerClient extends EphemeralClient { this.peerParty = await invite.establish({ mediaSrc, - hostParty: this.hostParty, - config: this.config, + role, + hostParty: role == 'host' ? this. this.hostParty : undefined, + model: role == 'client' ? this.hostParty.factory.model : undefined, rtcSettings: this.rtcSettings }) diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index eeb7ec0..feee206 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -58,7 +58,10 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = null + this.role = null + if(this.payload){ + this._updateRole() const expiry = this.payload.timestamp + DEFAULT_EXPIRY const now = Date.now() @@ -80,6 +83,18 @@ class PeerInvite extends EventEmitter { get to(){ return this.toIdentity } get from(){ return this.fromIdentity } + _updateRole(){ + + if(this.isSender()){ + + this.role = this.payload.role + return + + } + + this.role = this.payload.role == 'client' ? 'host' : 'client' + } + isSender(doc){ if(doc){ @@ -100,7 +115,7 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept(mediaSrc, config, hostParty){ + async accept({mediaSrc, model, hostParty, hostRunner}){ debug('accepting invite') /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ @@ -121,6 +136,7 @@ class PeerInvite extends EventEmitter { ) this.payload = payload.msg + this._updateRole() /*const expiry = this.payload.timestamp + DEFAULT_EXPIRY const now = Date.now() @@ -130,7 +146,7 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) }*/ - return await this.establish({mediaSrc, config, hostParty}) + return await this.establish({mediaSrc, model, hostParty, hostRunner}) } async reject(){ @@ -184,13 +200,13 @@ class PeerInvite extends EventEmitter { }) } - async establish({mediaSrc, hostParty, config, rtcSettings}){ + async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings}){ if(!rtcSettings){ rtcSettings = {} } - let host = this.isSender() + let host = (!this.isSender() && this.role == 'client') || (this.isSender() && this.role == 'host') let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from @@ -242,6 +258,10 @@ class PeerInvite extends EventEmitter { }*/ this.peerParty = new PeerParty({ + hostParty, + hostRunner, + model: hostParty.factory.model, + config: hostParty.config, comms: new RTCSocketComms({ host: this.isSender(), session: this.payload.session, @@ -258,9 +278,7 @@ class PeerInvite extends EventEmitter { trickle: rtcSettings.trickle? rtcSettings.trickle : true, discoverRemoteIdentity: false, remoteIdentity: otherIdentity - }), - hostParty: this.isSender() ? hostParty : undefined, - config: config ? config : hostParty.config + }) }) diff --git a/src/service/runner-router.js b/src/service/runner-router.js index c6897b8..24ae4dc 100644 --- a/src/service/runner-router.js +++ b/src/service/runner-router.js @@ -69,7 +69,7 @@ class RunnerRouter { * @returns {module:Service.ServiceRunner} */ getRunnerByHostIdentity(identity){ - const partyId = identity.toString() + const partyId = typeof identity !== 'string' ? identity.key.hash : identity debug('getRunnerByHostIdentity -', partyId) const runner = this.runnersByHost.get(partyId) @@ -83,7 +83,7 @@ class RunnerRouter { */ addRunner({domain, runner}){ - const partyId = runner.party.identity.toString() + const partyId = runner.party.identity.key.hash debug('addRunner - ', partyId, domain) if(!this.runnersByHost.has(partyId)){ diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index bebaded..01758ab 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -21,11 +21,39 @@ class ServiceHostPeer { } async onInvite(invite){ - // - //! check if service wants to allow user - const peerParty = await invite.accept(this.mediaSrc,this.config) + // Filter out none host mode requests + if(invite.role !== 'host'){ + debug('FAIL - unexpected role[', invite.role, '] we expecte to be host') + await invite.reject() + return + } + + let hostRunner = this.runner.party ? this.runner : this.runner.getRunnerByHostIdentity(invite.to) + + // Make sure we know the requested party & runner + if(!hostRunner){ + debug('FAIL - requested party not available', invite.to) + await invite.reject() + return + } + + //! Check if party wants to allow user + if(!(await hostRunner.auth.isSocketConnectionAllowed(invite.from))){ + debug('NOT ALLOWED - user is not allowed', invite.from) + await invite.reject() + return + } + + + let hostParty = hostRunner.party + + const peerParty = await invite.accept({ + media: this.mediaSrc, + hostParty, + hostRunner, + }) } } diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index 444c358..e98511e 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -12,8 +12,6 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) - //builder.addSchema(Path.join(__dirname, './schema/public-key.js')) - //builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) builder.addSchema(Path.join(__dirname, './schema/venue_project.js')) @@ -24,7 +22,6 @@ class VenueService extends DatapartySrv.IService { builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) builder.addMiddleware(DatapartySrv.middleware_paths.pre.ephemeral_session) - //builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) @@ -33,8 +30,6 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(DatapartySrv.endpoint_paths.version) builder.addEndpoint(DatapartySrv.endpoint_paths.key_announce) - //builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) - builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) From ca50083e1f8529713820a27f0be26dee95049ea3 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 16:45:15 +0000 Subject: [PATCH 37/50] matchmaker client identity --- src/party/peer/peer-invite.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index feee206..1ebe97b 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -98,11 +98,11 @@ class PeerInvite extends EventEmitter { isSender(doc){ if(doc){ - if(doc.toHash == matchMaker.identity.key.hash){return false } + if(doc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } - if(this.inviteDoc.toHash == matchMaker.identity.key.hash){return false } + if(this.inviteDoc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } From 32f600582e1faf3a77c0262625a9b35db416b3ae Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 17:13:20 +0000 Subject: [PATCH 38/50] plumbing model --- src/party/peer/ephemeral-client.js | 2 +- src/party/peer/peer-client.js | 12 +++++++++--- src/party/peer/peer-invite.js | 12 ++++++------ src/service/service-host-peer.js | 7 ++++--- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index ce36b7c..128a4ec 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -17,7 +17,7 @@ class EphemeralClient extends EventEmitter { this.contacts = contacts this.sessionKey = null this.identity = identity - this.role = role || 'guest' + this.role = role || 'guest' //! todo/note - these are different from invite roles this.wsParty = null this.restParty = null diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index c14bc18..fb29fb6 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -2,9 +2,11 @@ const EphemeralClient = require("./ephemeral-client") class PeerClient extends EphemeralClient { - constructor({config, hostParty, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + constructor({model=null, /*hostParty=null*/, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ - this.config = config + super({identity, role}) + + this.model = model this.hostParty = hostParty this.matchMaker = matchMaker @@ -20,6 +22,10 @@ class PeerClient extends EphemeralClient { this.rtcSettings = this.rtcSettings this.peerParty = null + + /*if(role == 'host' && hostParty==null){ + throw + }*/ } async start(mediaSrc){ @@ -41,7 +47,7 @@ class PeerClient extends EphemeralClient { mediaSrc, role, hostParty: role == 'host' ? this. this.hostParty : undefined, - model: role == 'client' ? this.hostParty.factory.model : undefined, + model: role == 'host' ? this.hostParty.factory.model : this.model, rtcSettings: this.rtcSettings }) diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index 1ebe97b..f51c834 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -115,7 +115,7 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept({mediaSrc, model, hostParty, hostRunner}){ + async accept({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity=false}){ debug('accepting invite') /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ @@ -146,7 +146,7 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) }*/ - return await this.establish({mediaSrc, model, hostParty, hostRunner}) + return await this.establish({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity}) } async reject(){ @@ -200,13 +200,13 @@ class PeerInvite extends EventEmitter { }) } - async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings}){ + async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings, discoverRemoteIdentity=false}){ if(!rtcSettings){ rtcSettings = {} } - let host = (!this.isSender() && this.role == 'client') || (this.isSender() && this.role == 'host') + let host = (this.role == 'host') let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from @@ -276,8 +276,8 @@ class PeerInvite extends EventEmitter { config: DEFAULT_ICE_SERVERS }, trickle: rtcSettings.trickle? rtcSettings.trickle : true, - discoverRemoteIdentity: false, - remoteIdentity: otherIdentity + discoverRemoteIdentity: discoverRemoteIdentity ? discoverRemoteIdentity : false, + remoteIdentity: discoverRemoteIdentity ? undefined: otherIdentity }) }) diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index 01758ab..f455bfe 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -5,13 +5,13 @@ class ServiceHostPeer { constructor({ runner, matchMaker, - mediaSrc - + mediaSrc, + discoverRemoteIdentity = false }){ this.runner = runner this.matchMaker = matchMaker - this.mediaSrc = mediaSrc + this.discoverRemoteIdentity = discoverRemoteIdentity } async start(){ @@ -53,6 +53,7 @@ class ServiceHostPeer { media: this.mediaSrc, hostParty, hostRunner, + discoverRemoteIdentity: this.discoverRemoteIdentity }) } } From c116a773824accfd3edfc793764e255b2f7b26ce Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 18:02:24 +0000 Subject: [PATCH 39/50] model, socket, internal calls plumbing --- src/party/document-factory.js | 2 +- src/party/peer/ephemeral-client.js | 51 +++++++++++++++++++++++++-- src/party/peer/peer-client.js | 15 ++++++-- src/party/peer/peer-invite.js | 2 +- src/service/iauth.js | 4 +++ src/service/middleware/pre/decrypt.js | 5 ++- src/service/service-host-peer.js | 12 ++++--- src/service/service-runner-node.js | 33 +++++++++++++++++ 8 files changed, 111 insertions(+), 13 deletions(-) diff --git a/src/party/document-factory.js b/src/party/document-factory.js index ba5c85f..6f7adf8 100644 --- a/src/party/document-factory.js +++ b/src/party/document-factory.js @@ -19,7 +19,7 @@ class DocumentFactory { this.factories = factories || {} this.party = party || null this.ajv = new Ajv() - //this.model = model + this.model = model this.documentClass = documentClass || IDocument this.validators = {} diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 128a4ec..a60f7f3 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -37,6 +37,14 @@ class EphemeralClient extends EventEmitter { } + get restParty(){ + return this.restParty + } + + get socketParty(){ + return this.wsParty + } + async start(){ this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) @@ -101,6 +109,43 @@ class EphemeralClient extends EventEmitter { } + async createSessionAnnoucement(){ + let currentActor = this.identity + + const announceData = { + annoucement: { + role: this.role, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + return announceData + } async announcePublicKeys(callPath='key/announce'){ @@ -165,7 +210,7 @@ class EphemeralClient extends EventEmitter { const lookupData = { hash } - const lookupResult = await this.wsParty.comms.call('key/lookup', lookupData, { + const lookupResult = await this.socketParty.comms.call('key/lookup', lookupData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -192,7 +237,7 @@ class EphemeralClient extends EventEmitter { expiry: !expiry ? Date.now()+24*60*60*3 : expiry } - const result = await this.wsParty.comms.call('short-code/create', request, { + const result = await this.socketParty.comms.call('short-code/create', request, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -212,7 +257,7 @@ class EphemeralClient extends EventEmitter { const request = { code } - const result = await this.wsParty.comms.call('short-code/lookup', request, { + const result = await this.socketParty.comms.call('short-code/lookup', request, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index fb29fb6..704f6ab 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -2,9 +2,9 @@ const EphemeralClient = require("./ephemeral-client") class PeerClient extends EphemeralClient { - constructor({model=null, /*hostParty=null*/, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + constructor({model=null, /*hostParty=null, */ contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ - super({identity, role}) + super({identity, contacts, role}) this.model = model this.hostParty = hostParty @@ -12,9 +12,10 @@ class PeerClient extends EphemeralClient { this.remoteIdentityHash = remoteIdentityHash + this.inviteSettings = { type: 'webrtc', - service: service, + service: model ? model.package.name : service, role: role ? role : 'client', session: null } @@ -28,6 +29,14 @@ class PeerClient extends EphemeralClient { }*/ } + get restParty(){ + return this.peerParty + } + + get socketParty(){ + return this.peerParty + } + async start(mediaSrc){ // diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index f51c834..59bfa9f 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -210,7 +210,7 @@ class PeerInvite extends EventEmitter { let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from - let party = this.this.matchMaker.client.socketPeerParty + let party = this.this.matchMaker.client.socketParty this.topicSub = new party.ROSLIB.Topic({ ros : party.comms.ros, diff --git a/src/service/iauth.js b/src/service/iauth.js index c83ef0e..7201d3f 100644 --- a/src/service/iauth.js +++ b/src/service/iauth.js @@ -37,6 +37,10 @@ module.exports = class IAuth { return identity } + async isPeerConnectionAllowed(identity){ + return true + } + async isSocketConnectionAllowed(identity){ //throw new Error('not implemented') return true diff --git a/src/service/middleware/pre/decrypt.js b/src/service/middleware/pre/decrypt.js index 048dd52..195b521 100644 --- a/src/service/middleware/pre/decrypt.js +++ b/src/service/middleware/pre/decrypt.js @@ -35,7 +35,10 @@ module.exports = class Decrypt extends IMiddleware { if(!context.input || !context.input.enc){ - if(!context.req.source || context.req.source != 'PeerComms'){ + if(!context.req.source || + ( context.req.source != 'PeerComms' && + context.req.source != 'INTERNAL' ) + ){ throw new Error('insecure message -' + context.req.source) } diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index f455bfe..ae67f5d 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -39,8 +39,12 @@ class ServiceHostPeer { return } - //! Check if party wants to allow user - if(!(await hostRunner.auth.isSocketConnectionAllowed(invite.from))){ + //! Validate the session annoucement payload + + invite.payload.info + + //! Check if party wants to allow / deny invite sender + if(!(await hostRunner.auth.isPeerConnectionAllowed(invite.from))){ debug('NOT ALLOWED - user is not allowed', invite.from) await invite.reject() return @@ -50,10 +54,10 @@ class ServiceHostPeer { let hostParty = hostRunner.party const peerParty = await invite.accept({ - media: this.mediaSrc, + mediaSrc: this.mediaSrc, hostParty, hostRunner, - discoverRemoteIdentity: this.discoverRemoteIdentity + discoverRemoteIdentity: this.discoverRemoteIdentity //! todo - this is probably something a party config could reasonably over ride }) } } diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index b251c2e..cb15e11 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -5,6 +5,7 @@ const Debug = require('debug') const debug = Debug('dataparty.service.runner-node') const EndpointContext = require('./endpoint-context') const DeltaTime = require('../utils/delta-time') +const HttpMocks = require('node-mocks-http') const Router = require('origin-router').Router const Runner = require('@dataparty/tasker').Runner @@ -446,6 +447,38 @@ class ServiceRunnerNode { } } + async internalRequest(endpoint, data){ + let bodyValue = data + + const req = HttpMocks.createRequest({ + method: 'GET', + url: '/'+endpoint, + body: bodyValue + }) + + const res = HttpMocks.createResponse() + + debug('\tthe request', req) + + debug('req ip type', typeof req.ip) + + const route = this.router.get(endpoint) + + debug('route',route) + + req.runner = this + req.source = 'INTERNAL' + + debug('call route', await route._events.route({ + method: req.method, + pathname: req.url, + request: req, + response: res + })) + + return {result: res._getData() } + } + async getTopic(path){ debug('looking up topic', path) From fc84be8f11db359cbf41b908e6e50f74ec63a7f9 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 19:07:52 +0000 Subject: [PATCH 40/50] plumb access controls better --- src/service/endpoints/key-announce.js | 21 ++++++++++++++------- src/service/iauth.js | 1 + src/service/middleware/post/encrypt.js | 5 ++++- src/service/service-host-peer.js | 25 ++++++++++++++++++++----- src/venue/endpoints/create-project.js | 15 ++++++++------- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index 60e8ffe..ed89db9 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -95,10 +95,16 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { } - // ensure sender is connected using session key mentioned in annoucement + // ensure sender is connected using session key mentioned in annoucement OR this is an internal call if(computedSessionHash == inputSessionKey.hash && - inputSessionKey.public.sign == ctx.senderKey.public.sign && - inputSessionKey.public.box == ctx.senderKey.public.box + ( + ctx.req.source == 'INTERNAL' || + //ctx.req.source == 'PeerComms' || + ( + inputSessionKey.public.sign == ctx.senderKey.public.sign && + inputSessionKey.public.box == ctx.senderKey.public.box + ) + ) ){ const actorSigBson = Routines.Utils.base64.decode( ctx.input.trust.actorSig ) @@ -124,10 +130,11 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { await actorSigMsg.assertVerified( actorIdentity, true ) await sessionSigMsg.assertVerified( sessionIdentity, true ) - // verify actor is an admin - const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) - if(!isAdmin){ - ctx.debug('non-admin user') + // verify actor is allowed + const isAllowed = (await ctx.runner.auth.isAdmin(actorIdentity)) || + (await ctx.runner.auth.isSocketConnectionAllowed(actrIdentity)) + if(!isAllowed){ + ctx.debug('non-allowed user') return {done: false} } diff --git a/src/service/iauth.js b/src/service/iauth.js index 7201d3f..5db6e17 100644 --- a/src/service/iauth.js +++ b/src/service/iauth.js @@ -46,6 +46,7 @@ module.exports = class IAuth { return true } + async isInternal(identity){ return false } diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index 8645087..2487722 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -31,7 +31,10 @@ module.exports = class Encrypt extends IMiddleware { if (!Config){ return } - if(ctx.req.source && ctx.req.source == 'PeerComms'){ + if(!context.req.source && + ( context.req.source == 'PeerComms' || + context.req.source == 'INTERNAL' ) + ){ ctx.setOutput(ctx.output) return } diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index ae67f5d..4aa5f9c 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -39,18 +39,33 @@ class ServiceHostPeer { return } - //! Validate the session annoucement payload - - invite.payload.info - - //! Check if party wants to allow / deny invite sender + //! Check if party wants to allow/deny invite sender if(!(await hostRunner.auth.isPeerConnectionAllowed(invite.from))){ + + // verify actor is allowed + const isAllowed = (await ctx.runner.auth.isAdmin(invite.from)) || + (await hostRunner.auth.isPeerConnectionAllowed(invite.from)) || + (await ctx.runner.auth.isSocketConnectionAllowed(invite.from)) + if(!isAllowed){ debug('NOT ALLOWED - user is not allowed', invite.from) await invite.reject() return } + //! Announce the session key + const annoucement = invite.payload.info + const result = await hostRunner.internalRequest('key/announce', annoucement) + + debug('annoucement result', result) + + if(!result || !result.done){ + + debug('user session not allowed') + + return + } + let hostParty = hostRunner.party const peerParty = await invite.accept({ diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index c763206..c4185be 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -1,5 +1,6 @@ const fs = require('fs') const Joi = require('joi') +const Path = require('path') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-project') @@ -103,7 +104,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { static async run(ctx){ - if(ctx.party.identity.key.hash != ctx.input.project.hash){ + if(ctx.party.identity.key.hash != ctx.input.project.venue){ throw new Error('project venue does not match this host') } @@ -176,7 +177,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('\t'+'hash', projectHash) - const projectWorkspace = 'projects'+safeFileName+'/'+ctx.input.project.version+'/'+safeProjectHash + const projectWorkspace = Path.join('projects/',safeFileName, ctx.input.project.version, safeProjectHash) const config = ctx.party.config @@ -195,11 +196,11 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('addProject', projectId) let projectDoc = (await ctx.party.find() - .type('venue_project') - .where('project.name').equals(project.name) - .where('project.version').equals(project.version) - .where('hash').equals(projectHash) - .exec())[0] + .type('venue_project') + .where('project.name').equals(project.name) + .where('project.version').equals(project.version) + .where('hash').equals(projectHash) + .exec())[0] if(!projectDoc){ From eea837515b93f294d1bad12d40d72b712f387b52 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 20:45:40 +0000 Subject: [PATCH 41/50] project staticTar handling --- src/comms/rest-comms.js | 14 ++++++++++- src/party/peer/ephemeral-client.js | 4 ++-- src/service/middleware/post/encrypt.js | 6 ++--- src/venue/bin/commands/project-build.js | 5 ++-- src/venue/bin/venue.js | 2 +- src/venue/endpoints/create-project.js | 32 ++++++++++++++++--------- src/venue/schema/venue_project.js | 2 +- 7 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/comms/rest-comms.js b/src/comms/rest-comms.js index 485e345..be11241 100644 --- a/src/comms/rest-comms.js +++ b/src/comms/rest-comms.js @@ -142,7 +142,19 @@ class RestComms extends EventEmitter { // debug('raw reply ->', reply) } catch (error) { debug('rest', fullPath, ' call fail ->', error.message) - throw new Error('RestCommsError') + + console.log(Object.keys(error), Object.keys(error.response)) + + const simpleError = { + name: error.name, + code: error.code, + //message: error.message, + statusCode: error.response.statusCode, + statusMessage: error.response.statusMessage, + data: error.response.data + } + + throw simpleError } const msg = await this.party.decrypt( diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index a60f7f3..8babd30 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -37,9 +37,9 @@ class EphemeralClient extends EventEmitter { } - get restParty(){ + /*get restParty(){ return this.restParty - } + }*/ get socketParty(){ return this.wsParty diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index 2487722..09dbe3d 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -31,9 +31,9 @@ module.exports = class Encrypt extends IMiddleware { if (!Config){ return } - if(!context.req.source && - ( context.req.source == 'PeerComms' || - context.req.source == 'INTERNAL' ) + if(!ctx.req.source && + ( ctx.req.source == 'PeerComms' || + ctx.req.source == 'INTERNAL' ) ){ ctx.setOutput(ctx.output) return diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 3708280..5d62c74 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -178,7 +178,7 @@ class VenueProjectBuild extends CmdTree.Command { const project = { owner: key.key.hash, - created: Date.now(), + //created: Date.now(), name: parsed.name ? parsed.name : projectJson.name, version: parsed.version ? parsed.version : projectJson.version, @@ -197,6 +197,7 @@ class VenueProjectBuild extends CmdTree.Command { const buildOutput = parsed.output+'/'+ project.name.replace('/', '-') +'.project.venue.json' + let prjFiles = [] prjFiles.push(buildOutput) @@ -226,7 +227,7 @@ class VenueProjectBuild extends CmdTree.Command { let staticTar = undefined - if(prjFiles.length == 3){ + if(prjFiles.length == 2){ staticTar = fs.readFileSync(prjFiles[ prjFiles.length - 1 ]) } diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index b405c7a..c83b6c0 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -123,7 +123,7 @@ async function main(){ context = { secureConfig, collectPassword, - exiting: false + exiting: true } const output = await commandTree.run({context}) diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index c4185be..c81d3f5 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -38,7 +38,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { validate: Joi.object().keys({ project: Joi.object().keys({ owner: Joi.string().required(), - created: Joi.number(), + //created: Joi.number(), name: Joi.string().required(), version: Joi.string().required(), @@ -149,19 +149,27 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('verified package signature') - const tarHash = Routines.Utils.hash(ctx.input.staticTar) - const tarHash64 = Routines.Utils.base64.encode( tarHash ) + const safeFileName = ctx.input.project.name.replace('/', '-') const tarFileName = safeFileName+'.files.venue.tgz' const projectFiles = ctx.input.project.files[tarFileName] - if(projectFiles && projectFiles.hash != tarHash64){ - throw new Error("staticTar hash doesn't match project definition") + if(projectFiles && !ctx.input.staticTar){ + throw new Error('project definition lists a static tar but none was uploaded') } - debug('verified staticTar') + if(ctx.input.staticTar){ + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + if(projectFiles && projectFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match project definition") + } + + debug('verified staticTar') + } debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) @@ -206,7 +214,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { if(!projectDoc){ debug('creating project') - const {owner, ...pkgWithoutOwner} = project.package + const {owner, ...pkgWithoutOwner} = project projectDoc = await ctx.party.createDocument('venue_pkg', { owner: project.owner, @@ -221,10 +229,12 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('need to update project?') } - fs.writeFileSync( - Path.join(workspacePath, tarFileName), - ctx.input.staticTar - ) + if(ctx.input.staticTar){ + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + } /*fs.writeFileSync( Path.join(workspacePath, safeFileName+'.service.venue.bson'), diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index 0dfa83e..03583c4 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -24,7 +24,7 @@ class VenueProject extends ISchema { project: { owner: {type: String, required: true, index: true}, //public_key.key.hash - created: {type: Number, required: true}, + //created: {type: Number, required: true}, name: {type: String, required: true, index: true}, version: {type: String, required: true, index: true}, From 53d043708af988a2e28d534deeeb7ca063c361f4 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 00:16:23 +0000 Subject: [PATCH 42/50] tar handling in package --- src/venue/endpoints/create-package.js | 35 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 4eedd2d..90b2da1 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -1,5 +1,6 @@ const fs = require('fs') const Joi = require('joi') +const Path = require('path') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-package') @@ -184,19 +185,23 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('verified package signature') - const tarHash = Routines.Utils.hash(ctx.input.staticTar) - const tarHash64 = Routines.Utils.base64.encode( tarHash ) - const safeFileName = ctx.input.build.package.name.replace('/', '-') - const tarFileName = safeFileName+'.files.venue.tgz' - const buildFiles = ctx.input.build.files[tarFileName] + if(ctx.input.staticTar){ - if(buildFiles && buildFiles.hash != tarHash64){ - throw new Error("staticTar hash doesn't match package definition") - } + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const tarFileName = safeFileName+'.files.venue.tgz' - debug('verified staticTar') + const buildFiles = ctx.input.build.files[tarFileName] + + if(buildFiles && buildFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match package definition") + } + + debug('verified staticTar') + } debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) @@ -212,7 +217,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('\t'+'hash', buildHash) - const buildWorkspace = 'packages/'+safeFileName+'/'+ctx.input.build.package.version+'/'+safeBuildHash + const buildWorkspace = Path.join('packages', safeFileName, ctx.input.build.package.version, safeBuildHash) const config = ctx.party.config @@ -259,10 +264,12 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('need to update service?') } - fs.writeFileSync( - Path.join(workspacePath, tarFileName), - ctx.input.staticTar - ) + if(ctx.input.staticTar){ + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + } /*fs.writeFileSync( Path.join(workspacePath, safeFileName+'.service.venue.bson'), From fde4fb3eb705f632675c220fd8fe97e4c619e686 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 01:36:58 +0000 Subject: [PATCH 43/50] project flow fixes --- src/service/endpoints/key-announce.js | 2 +- src/venue/bin/commands/venued-host.js | 36 ++++++++++++++++++++++++++- src/venue/endpoints/create-project.js | 7 ++++-- src/venue/schema/venue_project.js | 2 +- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index ed89db9..26afb71 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -132,7 +132,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { // verify actor is allowed const isAllowed = (await ctx.runner.auth.isAdmin(actorIdentity)) || - (await ctx.runner.auth.isSocketConnectionAllowed(actrIdentity)) + (await ctx.runner.auth.isSocketConnectionAllowed(actorIdentity)) if(!isAllowed){ ctx.debug('non-allowed user') return {done: false} diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 2ef584c..4785d1c 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -274,7 +274,41 @@ class VenuedHost extends CmdTree.Command { console.log('\t', i2pAddress) } - //console.log(Path.join(__dirname,'../public')) + /** + * config section + * + * projects: { + * [name]: latest-hash + * } + */ + + const projects = await config.read('projects') + + if(projects){ + for(let name in projects){ + + const hash = projects[name] + console.log('\tloading project', name, hash) + + + + const project = (await party.find() + .type('venue_project') + .where('hash').equals(hash).exec())[0] + + + + //console.log(project.data) + + /** + * 1. load project doc from db + * 2. validate files exist + * 3. launch project + */ + } + } + + this.context.exiting = false diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index c81d3f5..94f3699 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -216,9 +216,10 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { const {owner, ...pkgWithoutOwner} = project - projectDoc = await ctx.party.createDocument('venue_pkg', { + projectDoc = await ctx.party.createDocument('venue_project', { owner: project.owner, created: Date.now(), + changed: Date.now(), hash: projectHash, workspace: workspacePath, project: ctx.input.project, @@ -245,6 +246,8 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { Path.join(workspacePath, safeFileName+'.project.venue.json'), JSON.stringify(ctx.input.project, null, 2) ) + + await ctx.party.config.write('projects.'+ctx.input.project.name, projectHash) // verify build signature @@ -311,7 +314,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('updated service') }*/ - return {done: true} + return {done: true, project: projectDoc.data} //return {srv:projectDoc.data} } diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index 03583c4..77117c4 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -29,7 +29,7 @@ class VenueProject extends ISchema { name: {type: String, required: true, index: true}, version: {type: String, required: true, index: true}, venue: {type: String}, - domain: {type: String, index: true, unique: true}, + domain: {type: String, index: true/*, unique: true*/}, i2p: { address: String, From 44b8210d95d7fffe2b60d0dc457faaad09a15fb7 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 15:34:30 +0000 Subject: [PATCH 44/50] cleanup package and project creation --- src/bouncer/db/tingo-db.js | 7 ++- src/bouncer/mongo-query.js | 6 +- src/service/service-host-peer.js | 7 +-- src/venue/bin/commands/pkg-build.js | 8 ++- src/venue/bin/commands/project-build.js | 2 +- src/venue/endpoints/create-package.js | 6 +- src/venue/endpoints/create-project.js | 83 ++++++++++++++++++++++++- src/venue/example-project.js | 2 +- src/venue/schema/venue_package.js | 1 + src/venue/schema/venue_project.js | 1 + 10 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/bouncer/db/tingo-db.js b/src/bouncer/db/tingo-db.js index 69dcb4c..9ae83a7 100644 --- a/src/bouncer/db/tingo-db.js +++ b/src/bouncer/db/tingo-db.js @@ -164,10 +164,13 @@ module.exports = class TingoDb extends IDb { debug('find collection=', collectionName, ' query=', JSON.stringify(query,null,2)) let collection = await this.getCollection(collectionName) let cursor = await promisfy(collection.find.bind(collection))( - query, - mongoQuery.hasSort() ? mongoQuery.getSort() : undefined + query ) + if(mongoQuery.hasSort()){ + cursor = cursor.sort(mongoQuery.getSort()) + } + if(mongoQuery.hasLimit()){ cursor = cursor.limit(mongoQuery.getLimit()) } diff --git a/src/bouncer/mongo-query.js b/src/bouncer/mongo-query.js index ba09b97..1ad8111 100644 --- a/src/bouncer/mongo-query.js +++ b/src/bouncer/mongo-query.js @@ -44,7 +44,11 @@ class MongoQuery { } getSort () { - return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + if(Array.isArray(this.spec.sort.param)){ + return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + } + + return { [this.spec.sort.param]: this.spec.sort.direction } } /** diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index 4aa5f9c..03c37a1 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -40,12 +40,9 @@ class ServiceHostPeer { } //! Check if party wants to allow/deny invite sender - if(!(await hostRunner.auth.isPeerConnectionAllowed(invite.from))){ - - // verify actor is allowed - const isAllowed = (await ctx.runner.auth.isAdmin(invite.from)) || + const isAllowed = (await hostRunner.auth.isAdmin(invite.from)) || (await hostRunner.auth.isPeerConnectionAllowed(invite.from)) || - (await ctx.runner.auth.isSocketConnectionAllowed(invite.from)) + (await hostRunner.auth.isSocketConnectionAllowed(invite.from)) if(!isAllowed){ debug('NOT ALLOWED - user is not allowed', invite.from) await invite.reject() diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js index 5bcaca5..ee297ba 100644 --- a/src/venue/bin/commands/pkg-build.js +++ b/src/venue/bin/commands/pkg-build.js @@ -124,7 +124,13 @@ class VenuePackageBuild extends CmdTree.Command { if(parsed.deploy && parsed.remote){ console.log('uploading...') + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + if(!remote){ + throw 'invalid remote ['+parsed.remote+']' + } + let staticTar = undefined if(build.files.length == 3){ @@ -138,7 +144,7 @@ class VenuePackageBuild extends CmdTree.Command { } - async pushPackage(devId, remote, build, staticTar){ + async pushPackage(devId, remote, build, staticTar=null){ let client = new Dataparty.EphemeralClient({ identity: devId, diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 5d62c74..0a9d8eb 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -257,7 +257,7 @@ class VenueProjectBuild extends CmdTree.Command { } - async pushProject(devId, remote, build, staticTar){ + async pushProject(devId, remote, build, staticTar=null){ let client = new Dataparty.EphemeralClient({ identity: devId, diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 90b2da1..18d05d9 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -186,14 +186,13 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('verified package signature') const safeFileName = ctx.input.build.package.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' if(ctx.input.staticTar){ const tarHash = Routines.Utils.hash(ctx.input.staticTar) const tarHash64 = Routines.Utils.base64.encode( tarHash ) - const tarFileName = safeFileName+'.files.venue.tgz' - const buildFiles = ctx.input.build.files[tarFileName] if(buildFiles && buildFiles.hash != tarHash64){ @@ -254,6 +253,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { venue: ctx.party.identity.key.hash, hash: buildHash, workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), settings: ctx.input.settings, package: pkgWithoutOwner, compressedBuild: Routines.Utils.base64.encode(compressedBrotliBuild) @@ -346,7 +346,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('updated service') }*/ - return {done: true} + return {done: true, package: srvDoc.data} //return {srv:srvDoc.data} } diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index 94f3699..493185a 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -4,6 +4,9 @@ const Path = require('path') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-project') + +const process = require('process') +const tar = require('tar') const zlib = require('zlib') const IEndpoint = require('../../service/iendpoint') @@ -149,10 +152,8 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('verified package signature') - - const safeFileName = ctx.input.project.name.replace('/', '-') - const tarFileName = safeFileName+'.files.venue.tgz' + const tarFileName = safeFileName+'.project.files.venue.tgz' const projectFiles = ctx.input.project.files[tarFileName] @@ -171,6 +172,39 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('verified staticTar') } + // check route packages are valid + let packages = {} + let tarList = [] + + for(let route of ctx.input.project.routes){ + console.log(route) + let pkgDoc = (await ctx.party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + let pkgTarPath = pkgDoc.data.tarpath + + if(pkgTarPath && pkgTarPath.length > 0){ + tarList.push(pkgTarPath) + } + + packages[route.prefix] = pkgDoc.data + } + + debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) const projectBSON = Routines.BSON.serializeBSONWithoutOptimiser(projectWithoutSig) @@ -222,6 +256,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { changed: Date.now(), hash: projectHash, workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), project: ctx.input.project, }) @@ -230,11 +265,53 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('need to update project?') } + debug('extracting other tars', tarList) + + for(let tarPath of tarList){ + debug('extracting', tarPath) + + await tar.extract({ + cwd: workspacePath, + file: tarPath, + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, tarFileList ) + } + if(ctx.input.staticTar){ + debug('saving tar file', tarFileName) fs.writeFileSync( Path.join(workspacePath, tarFileName), ctx.input.staticTar ) + + debug('extracting contents') + + /*await tar.t({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + onReadEntry: entry => { console.log('\t\t', entry) } + })*/ + + //const tarFileInfo = projectDoc.data.project.files[ tarFileName ] + + //if(projectFiles){ + const tarFileList = Object.keys(projectFiles.files) + + debug('file list - ', tarFileList) + + await tar.extract({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, tarFileList ) + + //} } /*fs.writeFileSync( diff --git a/src/venue/example-project.js b/src/venue/example-project.js index 44b93e9..315dd1f 100644 --- a/src/venue/example-project.js +++ b/src/venue/example-project.js @@ -10,7 +10,7 @@ module.exports = { prefix: '/api', party:'SYSTEM', package: { - name: '@dataparty/venue' + name: '@dataparty/api' }, settings: { sendFullErrors: false, diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index e7802bd..8ce3914 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -18,6 +18,7 @@ class VenuePkg extends ISchema { changed: {type: Number}, venue: {type: String}, workspace: {type: String, required: true}, + tarpath: {type: String}, hash: {type: String, required: true, index: true}, settings: { //enabled: {type: Boolean, required: true}, diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index 77117c4..2d65721 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -17,6 +17,7 @@ class VenueProject extends ISchema { created: {type: Number, required: true}, changed: {type: Number, required: true}, workspace: {type: String, required: true}, + tarpath: {type: String}, hash: {type: String, required: true, index: true}, enabled: {type: Boolean}, From 3cd823298229ad13d6dafdfb684e978c79352eca Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 18:49:05 +0000 Subject: [PATCH 45/50] dynamic loading --- src/service/service-runner-node.js | 4 +- src/venue/bin/commands/venued-host.js | 112 +++++++++++++++++++++----- 2 files changed, 94 insertions(+), 22 deletions(-) diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index cb15e11..7d93b19 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -111,13 +111,13 @@ class ServiceRunnerNode { let AuthClass = null - if(!this.useNative){ + if(!this.useNative && Hoek.reach(this.service, `compiled.auth`)){ var self={} const build = Hoek.reach(this.service, `compiled.auth`) eval(build.code/*, build.map*/) AuthClass = self.Lib } - else{ + else if(this.service.constructors.auth){ AuthClass = this.service.constructors.auth } diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 4785d1c..0573a37 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -4,10 +4,12 @@ const debug = require('debug')('venued.host') const Path = require('path') const OS = require('os') const fs = require('fs') - +const zlib = require('zlib') const { execSync } = require('child_process') const Dataparty = require('../../../../') +const {Routines} = require('@dataparty/crypto') +const express = require('express') const HOMEDIR = OS.homedir() const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' @@ -191,7 +193,17 @@ class VenuedHost extends CmdTree.Command { CustomIpFilter.ips.push(ip) } } + + if(!fs.existsSync(parsed['ssl-key'])){ + execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', + {cwd: parsed.path} + ) + } + + const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') + const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') + const runner = new Dataparty.ServiceRunnerNode({ party, service, @@ -203,14 +215,6 @@ class VenuedHost extends CmdTree.Command { let runnerRouter = new Dataparty.RunnerRouter(runner) - if(!fs.existsSync(parsed['ssl-key'])){ - execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', - {cwd: parsed.path} - ) - } - - const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') - const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') @@ -287,24 +291,92 @@ class VenuedHost extends CmdTree.Command { if(projects){ for(let name in projects){ + let projectParties = {} + + //! todo - load parties and put them in projectParties map + const hash = projects[name] console.log('\tloading project', name, hash) - - const project = (await party.find() .type('venue_project') .where('hash').equals(hash).exec())[0] - - - //console.log(project.data) - - /** - * 1. load project doc from db - * 2. validate files exist - * 3. launch project - */ + const workspace = project.data.workspace + + for(let route of project.data.project.routes){ + console.log('route', route.package.name) + let pkgDoc = (await party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + const serviceFile = JSON.parse( + zlib.brotliDecompressSync( + Routines.Utils.base64.decode( compressedBuild ) + ) + ) + + console.log('decompressed', serviceFile.package) + + let serviceParty = null; + + + if(route.party == 'SYSTEM'){ + serviceParty = party + } else if( projectParties[route.party] ){ + serviceParty = projectParties[route.party] + } + + serviceParty.topics = new Dataparty.LocalTopicHost() + + debug('loading service') + const service = new Dataparty.IService(serviceFile.package, serviceFile) + debug('loaded service') + + let projectRunner = new Dataparty.ServiceRunnerNode({ + party: serviceParty, service, + sendFullErrors: route.settings.sendFullErrors, + useNative: route.settings.useNative, + prefix: route.prefix + }) + + if(route.party == 'SYSTEM'){ + //projectRunner.router = runner.router + } + + //await serviceParty.start() + await projectRunner.start() + + + console.log('workspace', workspace) + + const projectStaticPath = Path.join(workspace, 'public') + projectRunner.router.add('static-files', '/:path*', (req,res, next)=>{ + + let staticHandler = express.static(projectStaticPath) + + return staticHandler(req.request,req.response) + }) + + + await runnerRouter.addRunner({ + domain: project.data.project.domain, + runner: projectRunner + }) + } } } From c9b81e25ec01bdef09d9fbcb07d0768bd1a1d01b Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 19:37:49 +0000 Subject: [PATCH 46/50] static file hosting working --- src/venue/bin/commands/venued-host.js | 25 ++++++++++++++++++++----- src/venue/endpoints/create-project.js | 8 +++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 0573a37..81200cb 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -358,18 +358,33 @@ class VenuedHost extends CmdTree.Command { } //await serviceParty.start() - await projectRunner.start() + await projectRunner.start() console.log('workspace', workspace) const projectStaticPath = Path.join(workspace, 'public') - projectRunner.router.add('static-files', '/:path*', (req,res, next)=>{ + + let handler = (req,res)=>{ - let staticHandler = express.static(projectStaticPath) + let staticHandler = express.static(projectStaticPath, { index: ['index.html']}) - return staticHandler(req.request,req.response) - }) + let results = staticHandler(req.request,req.response, req.request.next) + + console.log('returning results', Object.keys(req.request)) + + + + console.log('sent already - headers?', req.response.headersSent) + console.log('sent already - date?', req.response.sendDate) + console.log('sent already - outputSize?', req.response.outputSize) + + } + + projectRunner.router.add('static-files1', '/:path*', handler) + projectRunner.router.add('static-files2', '/', handler) + + await runnerRouter.addRunner({ diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index 493185a..83c1f2a 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -277,7 +277,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { unlink: true, uid: process.getuid(), gid: process.getgid() - }, tarFileList ) + }, /*tarFileList*/ ) } if(ctx.input.staticTar){ @@ -391,6 +391,12 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('updated service') }*/ + console.log('restarting in 3 seconds...') + setTimeout(()=>{ + process.exit(), + 3000 + }) + return {done: true, project: projectDoc.data} //return {srv:projectDoc.data} From fdad1c5f0f1236f9bfd2e11facffcea863f640d6 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 28 Jun 2026 00:32:32 +0000 Subject: [PATCH 47/50] venue remote repl --- src/venue/bin/commands/venue-remote-repl.js | 138 ++++++++++++++++++++ src/venue/bin/venue.js | 1 + 2 files changed, 139 insertions(+) create mode 100644 src/venue/bin/commands/venue-remote-repl.js diff --git a/src/venue/bin/commands/venue-remote-repl.js b/src/venue/bin/commands/venue-remote-repl.js new file mode 100644 index 0000000..118fd2d --- /dev/null +++ b/src/venue/bin/commands/venue-remote-repl.js @@ -0,0 +1,138 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-repl') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const repl = require("repl"); + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'admin identity', + require: true + } +} + + +class VenueRemoteRepl extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteRepl.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote repl' + } + + static get Definition(){ + return { + usage: `venue remote repl`, + description: 'Run a repl with access to a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + + client.on('connecting',(info)=>{ + console.log('connecting', info) + }) + + client.on('connected',(info)=>{ + console.log('connected', info) + }) + + client.on('disconnected',(info)=>{ + console.log('disconnected', info) + }) + + client.on('reconnected',(info)=>{ + console.log('reconnected', info) + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + + await client.start() + console.log('client started') + + + + const replSrv = repl.start({ + prompt: parsed.identity +'@'+ parsed.remote+'> ' + }) + + replSrv.context.remote = remote + replSrv.context.client = client + + + this.context.exiting = false + + return // {remote, client, replSrv} + } +} + +module.exports = VenueRemoteRepl + + diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index c83b6c0..7ce2db9 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -22,6 +22,7 @@ commandTree.addCommand(require('./commands/venue-remote-add')) commandTree.addCommand(require('./commands/venue-remote-list')) commandTree.addCommand(require('./commands/venue-remote-show')) commandTree.addCommand(require('./commands/venue-remote-check')) +commandTree.addCommand(require('./commands/venue-remote-repl')) commandTree.addCommand(require('./commands/pkg-build')) commandTree.addCommand(require('./commands/project-build')) From f27761e461d404bf5b7848bd7cad7d82f7820a23 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 28 Jun 2026 00:38:52 +0000 Subject: [PATCH 48/50] peer query handler --- src/comms/peer-comms.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index 4402f66..68a23b9 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -92,7 +92,7 @@ class PeerComms extends ISocketComms { let response = null let request = await this.decrypt( {data: message}, this.remoteIdentity ) - debug('handleHostCall', truncateString(request, 1024)) + debug('handleClientCall', truncateString(JSON.stringify(request, null, 2), 1024)) let inputValidated @@ -461,11 +461,15 @@ class PeerComms extends ISocketComms { async handleCallOp(op){ debug('peer-call', op.input.endpoint) + const actor = await this.party.hostRunner.auth.lookupIdentity(this.remoteIdentity) + if(this.party.hostRunner){ debug('calling runner') - if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + + + if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->', truncateString(op.input.data, 1024)) op.result = {result: await this.party.handleCall(op.input.data) } @@ -514,7 +518,7 @@ class PeerComms extends ISocketComms { op.setState(HostOp.STATES.Finished_Success) return - } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->',op.input.data) op.result = {result: await this.party.handleCall(op.input.data) } From 54935f9e7f1f1a87a24f921563e24cceeb0fec22 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 28 Jun 2026 00:55:45 +0000 Subject: [PATCH 49/50] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 671ac7f..6ccd4f3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@dataparty/api", "private": false, - "version": "1.2.25", + "version": "1.3.0", "main": "dist/dataparty.js", "frontend": "dist/dataparty-browser.js", "backend": "dist/dataparty.js", From 50f3021b2ced975702bb99b083cdd8ec608ac8d1 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 1 Jul 2026 02:33:21 +0000 Subject: [PATCH 50/50] code of conduct --- CODE_OF_CONDUCT.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..232486a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Contributor Covenant 3.0 Code of Conduct + +## Our Pledge + +We pledge to make our community welcoming, safe, and equitable for all. + +We are committed to fostering an environment that respects and promotes the dignity, rights, and contributions of all individuals, regardless of characteristics including race, ethnicity, caste, color, age, physical characteristics, neurodiversity, disability, sex or gender, gender identity or expression, sexual orientation, language, philosophy or religion, national or social origin, socio-economic position, level of education, or other status. The same privileges of participation are extended to everyone who participates in good faith and in accordance with this Covenant. + +## Encouraged Behaviors + +While acknowledging differences in social norms, we all strive to meet our community's expectations for positive behavior. We also understand that our words and actions may be interpreted differently than we intend based on culture, background, or native language. + +With these considerations in mind, we agree to behave mindfully toward each other and act in ways that center our shared values, including: + +1. Respecting the **purpose of our community**, our activities, and our ways of gathering. +2. Engaging **kindly and honestly** with others. +3. Respecting **different viewpoints** and experiences. +4. **Taking responsibility** for our actions and contributions. +5. Gracefully giving and accepting **constructive feedback**. +6. Committing to **repairing harm** when it occurs. +7. Behaving in other ways that promote and sustain the **well-being of our community**. + + +## Restricted Behaviors + +We agree to restrict the following behaviors in our community. Instances, threats, and promotion of these behaviors are violations of this Code of Conduct. + +1. **Harassment.** Violating explicitly expressed boundaries or engaging in unnecessary personal attention after any clear request to stop. +2. **Character attacks.** Making insulting, demeaning, or pejorative comments directed at a community member or group of people. +3. **Stereotyping or discrimination.** Characterizing anyone’s personality or behavior on the basis of immutable identities or traits. +4. **Sexualization.** Behaving in a way that would generally be considered inappropriately intimate in the context or purpose of the community. +5. **Violating confidentiality**. Sharing or acting on a person's or Dataparty LLC's personal or private information without their express permission. +6. **Endangerment.** Causing, encouraging, or threatening violence or other harm toward any person or group. +7. Behaving in other ways that **threaten the well-being** of our community. +8. **Gaslighting.** Denying, misdirecting, lying or other means of refusing accountability despite clear evidence. +9. **Distraction** Sustained disruption of events or communications. +10. **Triangulation** Unnecesarily involving 3rd parties in moderation decisions. + +### Other Restrictions + +1. **Misleading identity.** Impersonating someone else for any reason, or pretending to be someone else to evade enforcement actions. +2. **Failing to credit sources.** Not properly crediting the sources of content you contribute. +3. **Promotional materials**. Sharing marketing or other commercial content in a way that is outside the norms of the community. +4. **Irresponsible communication.** Failing to responsibly present content which includes, links or describes any other restricted behaviors. + + +## Reporting an Issue + +Tensions can occur between community members even when they are trying their best to collaborate. Not every conflict represents a code of conduct violation, and this Code of Conduct reinforces encouraged behaviors and norms that can help avoid conflicts and minimize harm. + +When an incident does occur, it is important to report it promptly. To report a possible violation, **contact Dataparty LLC Managing Members nullagent (https://www.nullagent.com/) or rekcahdam (https://www.rekcahdam.com/)** + +Community Moderators take reports of violations seriously and will make every effort to respond in a timely manner. They will investigate all reports of code of conduct violations, reviewing messages, logs, and recordings, or interviewing witnesses and other participants. Community Moderators will keep investigation and enforcement actions as transparent as possible while prioritizing safety and confidentiality. In order to honor these values, enforcement actions are carried out in private with the involved parties, but communicating to the whole community may be part of a mutually agreed upon resolution. + + +## Addressing and Repairing Harm + +**[NOTE: The remedies and repairs outlined below are suggestions based on best practices in code of conduct enforcement. If your community has its own established enforcement process, be sure to edit this section to describe your own policies.]** + +If an investigation by the Community Moderators finds that this Code of Conduct has been violated, the following enforcement ladder may be used to determine how best to repair harm, based on the incident's impact on the individuals involved and the community as a whole. Depending on the severity of a violation, lower rungs on the ladder may be skipped. + +1) Warning + 1) Event: A violation involving a single incident or series of incidents. + 2) Consequence: A private, written warning from the Community Moderators. + 3) Repair: Examples of repair include a private written apology, acknowledgement of responsibility, and seeking clarification on expectations. +2) Temporarily Limited Activities + 1) Event: A repeated incidence of a violation that previously resulted in a warning, or the first incidence of a more serious violation. + 2) Consequence: A private, written warning with a time-limited cooldown period designed to underscore the seriousness of the situation and give the community members involved time to process the incident. The cooldown period may be limited to particular communication channels or interactions with particular community members. + 3) Repair: Examples of repair may include making an apology, using the cooldown period to reflect on actions and impact, and being thoughtful about re-entering community spaces after the period is over. +3) Temporary Suspension + 1) Event: A pattern of repeated violation which the Community Moderators have tried to address with warnings, or a single serious violation. + 2) Consequence: A private written warning with conditions for return from suspension. In general, temporary suspensions give the person being suspended time to reflect upon their behavior and possible corrective actions. + 3) Repair: Examples of repair include respecting the spirit of the suspension, meeting the specified conditions for return, and being thoughtful about how to reintegrate with the community when the suspension is lifted. +4) Permanent Ban + 1) Event: A pattern of repeated code of conduct violations that other steps on the ladder have failed to resolve, or a violation so serious that the Community Moderators determine there is no way to keep the community safe with this person as a member. + 2) Consequence: Access to all community spaces, tools, and communication channels is removed. In general, permanent bans should be rarely used, should have strong reasoning behind them, and should only be resorted to if working through other remedies has failed to change the behavior. + 3) Repair: There is no possible repair in cases of this severity. + +This enforcement ladder is intended as a guideline. It does not limit the ability of Community Managers to use their discretion and judgment, in keeping with the best interests of our community. + + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public or other spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 3.0, permanently available at [https://www.contributor-covenant.org/version/3/0/](https://www.contributor-covenant.org/version/3/0/). + +Contributor Covenant is stewarded by the Organization for Ethical Source and licensed under CC BY-SA 4.0. To view a copy of this license, visit [https://creativecommons.org/licenses/by-sa/4.0/](https://creativecommons.org/licenses/by-sa/4.0/) + +For answers to common questions about Contributor Covenant, see the FAQ at [https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). Translations are provided at [https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). Additional enforcement and community guideline resources can be found at [https://www.contributor-covenant.org/resources](https://www.contributor-covenant.org/resources). The enforcement ladder was inspired by the work of [Mozilla’s code of conduct team](https://github.com/mozilla/inclusion).