diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..232486a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Contributor Covenant 3.0 Code of Conduct + +## Our Pledge + +We pledge to make our community welcoming, safe, and equitable for all. + +We are committed to fostering an environment that respects and promotes the dignity, rights, and contributions of all individuals, regardless of characteristics including race, ethnicity, caste, color, age, physical characteristics, neurodiversity, disability, sex or gender, gender identity or expression, sexual orientation, language, philosophy or religion, national or social origin, socio-economic position, level of education, or other status. The same privileges of participation are extended to everyone who participates in good faith and in accordance with this Covenant. + +## Encouraged Behaviors + +While acknowledging differences in social norms, we all strive to meet our community's expectations for positive behavior. We also understand that our words and actions may be interpreted differently than we intend based on culture, background, or native language. + +With these considerations in mind, we agree to behave mindfully toward each other and act in ways that center our shared values, including: + +1. Respecting the **purpose of our community**, our activities, and our ways of gathering. +2. Engaging **kindly and honestly** with others. +3. Respecting **different viewpoints** and experiences. +4. **Taking responsibility** for our actions and contributions. +5. Gracefully giving and accepting **constructive feedback**. +6. Committing to **repairing harm** when it occurs. +7. Behaving in other ways that promote and sustain the **well-being of our community**. + + +## Restricted Behaviors + +We agree to restrict the following behaviors in our community. Instances, threats, and promotion of these behaviors are violations of this Code of Conduct. + +1. **Harassment.** Violating explicitly expressed boundaries or engaging in unnecessary personal attention after any clear request to stop. +2. **Character attacks.** Making insulting, demeaning, or pejorative comments directed at a community member or group of people. +3. **Stereotyping or discrimination.** Characterizing anyone’s personality or behavior on the basis of immutable identities or traits. +4. **Sexualization.** Behaving in a way that would generally be considered inappropriately intimate in the context or purpose of the community. +5. **Violating confidentiality**. Sharing or acting on a person's or Dataparty LLC's personal or private information without their express permission. +6. **Endangerment.** Causing, encouraging, or threatening violence or other harm toward any person or group. +7. Behaving in other ways that **threaten the well-being** of our community. +8. **Gaslighting.** Denying, misdirecting, lying or other means of refusing accountability despite clear evidence. +9. **Distraction** Sustained disruption of events or communications. +10. **Triangulation** Unnecesarily involving 3rd parties in moderation decisions. + +### Other Restrictions + +1. **Misleading identity.** Impersonating someone else for any reason, or pretending to be someone else to evade enforcement actions. +2. **Failing to credit sources.** Not properly crediting the sources of content you contribute. +3. **Promotional materials**. Sharing marketing or other commercial content in a way that is outside the norms of the community. +4. **Irresponsible communication.** Failing to responsibly present content which includes, links or describes any other restricted behaviors. + + +## Reporting an Issue + +Tensions can occur between community members even when they are trying their best to collaborate. Not every conflict represents a code of conduct violation, and this Code of Conduct reinforces encouraged behaviors and norms that can help avoid conflicts and minimize harm. + +When an incident does occur, it is important to report it promptly. To report a possible violation, **contact Dataparty LLC Managing Members nullagent (https://www.nullagent.com/) or rekcahdam (https://www.rekcahdam.com/)** + +Community Moderators take reports of violations seriously and will make every effort to respond in a timely manner. They will investigate all reports of code of conduct violations, reviewing messages, logs, and recordings, or interviewing witnesses and other participants. Community Moderators will keep investigation and enforcement actions as transparent as possible while prioritizing safety and confidentiality. In order to honor these values, enforcement actions are carried out in private with the involved parties, but communicating to the whole community may be part of a mutually agreed upon resolution. + + +## Addressing and Repairing Harm + +**[NOTE: The remedies and repairs outlined below are suggestions based on best practices in code of conduct enforcement. If your community has its own established enforcement process, be sure to edit this section to describe your own policies.]** + +If an investigation by the Community Moderators finds that this Code of Conduct has been violated, the following enforcement ladder may be used to determine how best to repair harm, based on the incident's impact on the individuals involved and the community as a whole. Depending on the severity of a violation, lower rungs on the ladder may be skipped. + +1) Warning + 1) Event: A violation involving a single incident or series of incidents. + 2) Consequence: A private, written warning from the Community Moderators. + 3) Repair: Examples of repair include a private written apology, acknowledgement of responsibility, and seeking clarification on expectations. +2) Temporarily Limited Activities + 1) Event: A repeated incidence of a violation that previously resulted in a warning, or the first incidence of a more serious violation. + 2) Consequence: A private, written warning with a time-limited cooldown period designed to underscore the seriousness of the situation and give the community members involved time to process the incident. The cooldown period may be limited to particular communication channels or interactions with particular community members. + 3) Repair: Examples of repair may include making an apology, using the cooldown period to reflect on actions and impact, and being thoughtful about re-entering community spaces after the period is over. +3) Temporary Suspension + 1) Event: A pattern of repeated violation which the Community Moderators have tried to address with warnings, or a single serious violation. + 2) Consequence: A private written warning with conditions for return from suspension. In general, temporary suspensions give the person being suspended time to reflect upon their behavior and possible corrective actions. + 3) Repair: Examples of repair include respecting the spirit of the suspension, meeting the specified conditions for return, and being thoughtful about how to reintegrate with the community when the suspension is lifted. +4) Permanent Ban + 1) Event: A pattern of repeated code of conduct violations that other steps on the ladder have failed to resolve, or a violation so serious that the Community Moderators determine there is no way to keep the community safe with this person as a member. + 2) Consequence: Access to all community spaces, tools, and communication channels is removed. In general, permanent bans should be rarely used, should have strong reasoning behind them, and should only be resorted to if working through other remedies has failed to change the behavior. + 3) Repair: There is no possible repair in cases of this severity. + +This enforcement ladder is intended as a guideline. It does not limit the ability of Community Managers to use their discretion and judgment, in keeping with the best interests of our community. + + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public or other spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 3.0, permanently available at [https://www.contributor-covenant.org/version/3/0/](https://www.contributor-covenant.org/version/3/0/). + +Contributor Covenant is stewarded by the Organization for Ethical Source and licensed under CC BY-SA 4.0. To view a copy of this license, visit [https://creativecommons.org/licenses/by-sa/4.0/](https://creativecommons.org/licenses/by-sa/4.0/) + +For answers to common questions about Contributor Covenant, see the FAQ at [https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). Translations are provided at [https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). Additional enforcement and community guideline resources can be found at [https://www.contributor-covenant.org/resources](https://www.contributor-covenant.org/resources). The enforcement ladder was inspired by the work of [Mozilla’s code of conduct team](https://github.com/mozilla/inclusion). diff --git a/examples/secure-config-argon2.js b/examples/secure-config-argon2.js index d4df83f..ce419d4 100644 --- a/examples/secure-config-argon2.js +++ b/examples/secure-config-argon2.js @@ -18,7 +18,6 @@ const HELP_INFO = ` ` async function main(){ - const memoryConfig = new Dataparty.Config.MemoryConfig({foo: 'bar'}) const jsonConfig = new Dataparty.Config.JsonFileConfig({ diff --git a/package.json b/package.json index 5caa37b..6ccd4f3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@dataparty/api", "private": false, - "version": "1.2.25", + "version": "1.3.0", "main": "dist/dataparty.js", "frontend": "dist/dataparty-browser.js", "backend": "dist/dataparty.js", @@ -47,6 +47,10 @@ "dist", "src/*" ], + "bin": { + "venue": "./src/venue/bin/venue.js", + "venued": "./src/venue/bin/venued.js" + }, "scripts": { "test": "npx lab", "build": "npx parcel build --no-scope-hoist", @@ -63,10 +67,10 @@ }, "dependencies": { "@babel/runtime": "^7.28.4", - "@dataparty/bouncer-db": "1.0.1", + "@dataparty/bouncer-db": "https://github.com/datapartyjs/bouncer-db#mongoose-latest", "@dataparty/crypto": "github:datapartyjs/dataparty-crypto", "@dataparty/tasker": "^0.0.3", - "@diva.exchange/i2p-sam": "^4.1.8", + "@diva.exchange/i2p-sam": "5.5.2", "@markwylde/liferaft": "^1.3.4", "@sevenbitbyte/ncc": "0.0.2", "ajv": "6.12.5", @@ -77,6 +81,7 @@ "buffer": "^6.0.3", "bufferutil": "^4.0.8", "colors": "1.3.1", + "command-tree": "github:datapartyjs/command-tree", "cors": "^2.8.5", "debug": "^3.1.0", "dom-storage": "^2.1.0", @@ -84,11 +89,14 @@ "express": "^4.17.1", "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", + "fast-safe-stringify": "^2.1.1", + "find-up-json": "^2.0.5", "git-repo-info": "^2.1.1", - "joi": "^17.13.3", + "glob": "^13.0.6", + "joi": "^18.2.1", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", - "jsonpath-plus": "^0.20.1", + "jsonpath-plus": "10.4.0", "last-eventemitter": "^1.1.1", "lodash": "^4.17.21", "lokijs": "1.5.12", @@ -100,7 +108,7 @@ "node-mocks-http": "^1.12.1", "node-object-hash": "^3.0.0", "node-persist": "^3.0.1", - "origin-router": "^1.6.4", + "origin-router": "https://github.com/datapartyjs/origin-router.git", "parse-url": "^5.0.1", "promisfy": "^1.2.0", "roslib": "^1.3.0", @@ -108,6 +116,7 @@ "simple-peer": "9.11.1", "source-map": "^0.7.3", "store-js": "^2.0.4", + "tar": "^7.5.15", "tingodb": "^0.6.1", "touch": "^3.1.0", "url-parse": "^1.4.7", @@ -118,8 +127,9 @@ "zangodb": "https://github.com/sevenbitbyte/zangodb#hash-patch" }, "devDependencies": { - "@dataparty/bouncer-model": "1.4.3", + "@dataparty/bouncer-model": "https://github.com/datapartyjs/bouncer-model#mongoose-latest", "@hapi/code": "^9.0.1", + "@hapi/joi": "^17.1.1", "@hapi/lab": "^25.0.1", "argon2": "^0.30.3", "argon2-browser": "^1.18.0", diff --git a/scripts/install-i2p-ubuntu-24.sh b/scripts/install-i2p-ubuntu-24.sh new file mode 100755 index 0000000..55019f1 --- /dev/null +++ b/scripts/install-i2p-ubuntu-24.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo add-apt-repository ppa:purplei2p/i2pd +sudo apt update +sudo apt install --yes i2pd + diff --git a/src/bouncer/db/tingo-db.js b/src/bouncer/db/tingo-db.js index 69dcb4c..9ae83a7 100644 --- a/src/bouncer/db/tingo-db.js +++ b/src/bouncer/db/tingo-db.js @@ -164,10 +164,13 @@ module.exports = class TingoDb extends IDb { debug('find collection=', collectionName, ' query=', JSON.stringify(query,null,2)) let collection = await this.getCollection(collectionName) let cursor = await promisfy(collection.find.bind(collection))( - query, - mongoQuery.hasSort() ? mongoQuery.getSort() : undefined + query ) + if(mongoQuery.hasSort()){ + cursor = cursor.sort(mongoQuery.getSort()) + } + if(mongoQuery.hasLimit()){ cursor = cursor.limit(mongoQuery.getLimit()) } diff --git a/src/bouncer/ischema.js b/src/bouncer/ischema.js index 78fd0a8..18c080b 100644 --- a/src/bouncer/ischema.js +++ b/src/bouncer/ischema.js @@ -1,4 +1,4 @@ -const debug = require('debug')('bouncer.ISchema') +const debug = require('debug')('dataparty.bouncer.ISchema') const MgoUtils = require('../utils/mongoose-scheme-utils') module.exports = class ISchema { diff --git a/src/bouncer/mongo-query.js b/src/bouncer/mongo-query.js index ba09b97..1ad8111 100644 --- a/src/bouncer/mongo-query.js +++ b/src/bouncer/mongo-query.js @@ -44,7 +44,11 @@ class MongoQuery { } getSort () { - return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + if(Array.isArray(this.spec.sort.param)){ + return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + } + + return { [this.spec.sort.param]: this.spec.sort.direction } } /** diff --git a/src/comms/isocket-comms.js b/src/comms/isocket-comms.js index 94070ae..cfe90fc 100644 --- a/src/comms/isocket-comms.js +++ b/src/comms/isocket-comms.js @@ -71,6 +71,7 @@ class ISocketComms extends EventEmitter { this.connected = false debug('Server closed connection') this.emit('close') + this.emit('server-close') } onopen(){ diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index c19a07c..68a23b9 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -54,6 +54,9 @@ class PeerComms extends ISocketComms { this.uuid = uuidv4() this.socket = socket || null + this.stopped = false + this.started = false + //this.auto_reconnect = !socket && !host this.host = host //! Is comms host\ this.oncall = null @@ -89,7 +92,7 @@ class PeerComms extends ISocketComms { let response = null let request = await this.decrypt( {data: message}, this.remoteIdentity ) - debug('handleHostCall', truncateString(request, 1024)) + debug('handleClientCall', truncateString(JSON.stringify(request, null, 2), 1024)) let inputValidated @@ -203,11 +206,16 @@ class PeerComms extends ISocketComms { async start(){ debug('start') + + if(this.started){ return } + + this.started = true + if(this.socketInit){ await this.socketInit() } - this.socket.on('close', this.stop.bind(this)) + this.socket.on('close', this.socketStop.bind(this)) if(this.host){ debug('host mode comms') @@ -226,7 +234,14 @@ class PeerComms extends ISocketComms { } } + socketStop(){ + debug('socket stop') + this.close() + } + async stop(){ + this.stopped = true + this.started = false debug('stop') this.close() } @@ -385,13 +400,13 @@ class PeerComms extends ISocketComms { if(this.party.hostRunner){ const actor = await this.party.hostRunner.auth.lookupIdentity(offer.sender) - const verified = await Routines.verifyDataPQ(actor, signature, offerBSON) + const verified = await Routines.verifyDataPQ(offer.sender, signature, offerBSON) if(!verified){ throw new Error('DENY(hostRunner) - auth op signature is not valid') } - if(this.discoverRemoteIdentity){ this.remoteIdentity = actor } + if(this.discoverRemoteIdentity){ this.remoteIdentity = offer.sender } const authorized = await this.party.hostRunner.auth.isSocketConnectionAllowed(actor) if(!authorized){ @@ -406,6 +421,7 @@ class PeerComms extends ISocketComms { await this.stop() debug('DENY - client not allowed - ', this.remoteIdentity) + throw new Error('DENY - client not allowed') } } else { const actor = offer.sender @@ -420,7 +436,7 @@ class PeerComms extends ISocketComms { } } - debug('clienr auth op offer -', offer) + debug('client auth op offer -', offer) debug('ALLOW - allowing client - ', this.remoteIdentity) this.aesStream = await AESStream.recoverStream( @@ -445,11 +461,15 @@ class PeerComms extends ISocketComms { async handleCallOp(op){ debug('peer-call', op.input.endpoint) + const actor = await this.party.hostRunner.auth.lookupIdentity(this.remoteIdentity) + if(this.party.hostRunner){ debug('calling runner') - if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + + + if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->', truncateString(op.input.data, 1024)) op.result = {result: await this.party.handleCall(op.input.data) } @@ -457,10 +477,18 @@ class PeerComms extends ISocketComms { return } + debug('input type', typeof op.input.data, Object.keys(op.input.data)) + debug('op.msg type', typeof op.msg, Object.keys(op.msg), Buffer.isBuffer(op.msg)) + + let bodyValue = Buffer.isBuffer(op.msg) ? + op.input.data : + //Routines.BSON.parseObject(new Routines.BSON.BaseParser( op.msg )) : + JSON.parse(op.msg.toString()) + const req = HttpMocks.createRequest({ method: 'GET', url: '/'+op.input.endpoint, - body: (op.input.data) ? JSON.parse(op.msg.toString()) : undefined + body: bodyValue }) const res = HttpMocks.createResponse() @@ -473,6 +501,9 @@ class PeerComms extends ISocketComms { debug('route',route) + req.peer = this + req.source = 'PeerComms' + debug('call route', await route._events.route({ method: req.method, pathname: req.url, @@ -487,7 +518,7 @@ class PeerComms extends ISocketComms { op.setState(HostOp.STATES.Finished_Success) return - } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->',op.input.data) op.result = {result: await this.party.handleCall(op.input.data) } diff --git a/src/comms/rest-comms.js b/src/comms/rest-comms.js index cac3a67..be11241 100644 --- a/src/comms/rest-comms.js +++ b/src/comms/rest-comms.js @@ -142,7 +142,19 @@ class RestComms extends EventEmitter { // debug('raw reply ->', reply) } catch (error) { debug('rest', fullPath, ' call fail ->', error.message) - throw new Error('RestCommsError') + + console.log(Object.keys(error), Object.keys(error.response)) + + const simpleError = { + name: error.name, + code: error.code, + //message: error.message, + statusCode: error.response.statusCode, + statusMessage: error.response.statusMessage, + data: error.response.data + } + + throw simpleError } const msg = await this.party.decrypt( @@ -207,7 +219,7 @@ class RestComms extends EventEmitter { const serverIdentity = await RestComms.HttpGet(this.uri + `${this.uriPrefix}identity`) debug('server identity - ', serverIdentity) - this.remoteIdentity = new dataparty_crypto.Identity(serverIdentity) + this.remoteIdentity = dataparty_crypto.Identity.fromJSON(serverIdentity) } return this.remoteIdentity diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index d91d690..fbfce40 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -14,11 +14,13 @@ const WebsocketShim = require('./websocket-shim') * @see https://en.wikipedia.org/wiki/WebSocket */ class WebsocketComms extends PeerComms { - constructor({uri, connection, remoteIdentity, host, party, ...options}){ + constructor({uri, connection, timeout=20000, remoteIdentity, host, party, ...options}){ super({remoteIdentity, host, party, ...options}) this.uri = uri this.connection = connection + this.timeout = timeout + this.timer = null debug('starting host=',host, ' uuid=', this.uuid, ' uri=', this.uri) @@ -34,13 +36,44 @@ class WebsocketComms extends PeerComms { async socketInit(){ debug('init') - + let isNewConnection = false + if(!this.host && !this.connection){ debug('opening client connection to',this.uri) this.connection = new WebSocket(this.uri) + + isNewConnection = true } this.socket = new WebsocketShim(this.connection) + + if(isNewConnection){ + + //await new Promise((resolve,reject)=>{ + this.timer = setTimeout(() => { + debug('websocket timeout') + this.connection.close() + this.emit('timeout') + //reject(new Error("WebSocket connection timeout")); + }, this.timeout); + + this.socket.once('connect', () => { + debug('websocket opened') + clearTimeout(this.timer); + //resolve(); + }) + + this.socket.once('error',(error) => { + debug('websocket error', error) + clearTimeout(this.timer) + this.emit('error', error) + //this.connection.close() + //reject(error); + }) + //}) + } + + } } diff --git a/src/comms/websocket-shim.js b/src/comms/websocket-shim.js index 1f8b2ec..1a6e72a 100644 --- a/src/comms/websocket-shim.js +++ b/src/comms/websocket-shim.js @@ -19,7 +19,7 @@ class WebsocketShim extends EventEmitter { } this.conn.onclose = (event) => { - debug('onclose', event) + debug('onclose', event, event.code, event.reason) this.emit('close', event) } @@ -39,6 +39,7 @@ class WebsocketShim extends EventEmitter { } destroy(){ + if(this.conn && this.conn.terminate) this.conn.terminate() } diff --git a/src/config/json-file.js b/src/config/json-file.js index b5280ab..a8376c8 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -22,6 +22,8 @@ class JsonFileConfig extends IConfig { this.path = this.basePath +'/config.json' this.defaults = defaults || {} this.content = Object.assign({}, this.defaults) + this.writing = false + this.started = false } async load(){ @@ -47,9 +49,16 @@ class JsonFileConfig extends IConfig { } async start () { + + if(this.started){return} + await this.touchDir('') await this.load() + + fs.watchFile(this.path, this.handleFileChange.bind(this)) logger('started') + + this.started = true } async clear () { @@ -79,7 +88,9 @@ class JsonFileConfig extends IConfig { } async save(){ + this.writing = true fs.writeFileSync(this.path, JSON.stringify(this.content, null, 2)) + this.writing = false } async touchDir (path) { @@ -98,6 +109,24 @@ class JsonFileConfig extends IConfig { }) }) } + + fileExists(path){ + var realPath = Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + + return fs.existsSync(realPath) + } + + filePath(path){ + return Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + } + + async handleFileChange(current, previous){ + if(this.writing){ return } + + logger('config changed, reloading') + + await this.load() + } } module.exports = JsonFileConfig \ No newline at end of file diff --git a/src/party/document-factory.js b/src/party/document-factory.js index ba5c85f..6f7adf8 100644 --- a/src/party/document-factory.js +++ b/src/party/document-factory.js @@ -19,7 +19,7 @@ class DocumentFactory { this.factories = factories || {} this.party = party || null this.ajv = new Ajv() - //this.model = model + this.model = model this.documentClass = documentClass || IDocument this.validators = {} diff --git a/src/party/index-browser.js b/src/party/index-browser.js index cf40f10..b084914 100644 --- a/src/party/index-browser.js +++ b/src/party/index-browser.js @@ -7,6 +7,7 @@ const ZangoParty = require('./local/zango-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') const LokiDb = require('../bouncer/db/loki-db') @@ -15,5 +16,5 @@ module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, ZangoParty, PeerParty, - LokiDb, MatchMakerClient + LokiDb, EphemeralClient, MatchMakerClient } diff --git a/src/party/index-embedded.js b/src/party/index-embedded.js index 422f737..ba36e58 100644 --- a/src/party/index-embedded.js +++ b/src/party/index-embedded.js @@ -7,11 +7,12 @@ const TingoParty = require('./local/tingo-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, PeerParty, - TingoParty, MatchMakerClient + TingoParty, EphemeralClient, MatchMakerClient } \ No newline at end of file diff --git a/src/party/index.js b/src/party/index.js index e362eff..af6f1cc 100644 --- a/src/party/index.js +++ b/src/party/index.js @@ -9,4 +9,5 @@ exports.MongoParty = require('./mongo/mongo-party') exports.IDocument = require('./idocument') exports.DocumentFactory = require('./document-factory') exports.CloudDocument = require('./cloud/cloud-document') -exports.MatchMakerClient = require('./peer/match-maker-client') \ No newline at end of file +exports.EphemeralClient = require('./peer/ephemeral-client') +exports.MatchMakerClient = require('./peer/match-maker-client') diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js new file mode 100644 index 0000000..8babd30 --- /dev/null +++ b/src/party/peer/ephemeral-client.js @@ -0,0 +1,276 @@ +const EventEmitter = require('eventemitter3') + +const debug = require('debug')('dataparty.ephemeral-client') + +const dataparty_crypto = require('@dataparty/crypto') +const LokiParty = require('../local/loki-party') +const PeerParty = require('./peer-party') +const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') +const WebsocketComms = require('../../comms/websocket-comms') + +class EphemeralClient extends EventEmitter { + constructor({identity, role='guest', contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + + super() + + this.contacts = contacts + this.sessionKey = null + this.identity = identity + this.role = role || 'guest' //! todo/note - these are different from invite roles + this.wsParty = null + this.restParty = null + + if(typeof urlOrParty == 'string'){ + this.restUrl = urlOrParty + this.restParty = null + } else { + this.restParty = urlOrParty + } + + if(typeof wsUrlOrParty == 'string'){ + this.wsUrl = wsUrlOrParty + this.wsParty = null + } else { + this.wsParty = wsUrlOrParty + } + } + + + /*get restParty(){ + return this.restParty + }*/ + + get socketParty(){ + return this.wsParty + } + + + async start(){ + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + if(!this.restParty){ + let config = new MemoryConfig({ + basePath:'ephemeral-client', + cloud: { + uri: this.restUrl + } + }) + + this.restParty = new LokiParty({ + path: 'ephemeral-client', + dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), + config + }) + + await this.restParty.setIdentity(this.sessionKey) + + debug('starting restParty') + await this.restParty.start() + + if(!this.restParty.comms){ + this.restParty.comms = new RestComms({ + party:this.restParty, + config: this.restParty.config + }) + + this.restParty.comms.sessionId = this.sessionKey.key.hash + } + + await this.announcePublicKeys() + } + + if(!this.wsParty && this.wsUrl){ + + this.wsParty = new PeerParty({ + comms: new WebsocketComms({ + uri: this.wsUrl, + discoverRemoteIdentity: false, + remoteIdentity: await this.restParty.comms.getServiceIdentity(), + session: this.sessionKey.key.hash + }), + config: this.restParty.config + }) + + this.wsParty.comms.on('close', ()=>{ + + let stopped = this.wsParty.comms.stopped + console.log('hey the ws closed - stopped=', stopped) + + }) + + await this.wsParty.start() + + debug('starting wsParty') + await this.wsParty.start() + debug('waiting for websocket authorization') + await this.wsParty.comms.authorized() + } + + } + + async createSessionAnnoucement(){ + let currentActor = this.identity + + const announceData = { + annoucement: { + role: this.role, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + return announceData + } + + async announcePublicKeys(callPath='key/announce'){ + + let currentActor = this.identity + + const announceData = { + annoucement: { + role: this.role, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + debug('announcePublicKeys', announceData) + + const announceResult = await this.restParty.comms.call(callPath, announceData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: false + }) + + if(announceResult.done != true){ + throw new Error('annoucement request failed - '+callPath) + } + } + + + async lookupPublicKey(hash){ + debug('lookupPublicKey - hash:', hash) + + if(hash == this.identity.key.hash){ + return this.identity + } + + if(this.contacts){ + return await this.contacts.lookupPublicKey(hash) + } + + const lookupData = { hash } + + const lookupResult = await this.socketParty.comms.call('key/lookup', lookupData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + if(!lookupResult.done){ + return null + } + + debug('lookup result -', lookupResult) + + const identity = new dataparty_crypto.Identity({ + key: lookupResult.public_key + }) + + return identity + } + + async createShortCode(use_limit=3, expiry){ + debug('createShortCode') + + const request = { + use_limit, + expiry: !expiry ? Date.now()+24*60*60*3 : expiry + } + + const result = await this.socketParty.comms.call('short-code/create', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('createShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } + + async lookupPublicKeyByShortCode( code ){ + debug('lookupPublicKeyByShortCode') + + const request = { code } + + const result = await this.socketParty.comms.call('short-code/lookup', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('lookupPublicKeyByShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } +} + +module.exports = EphemeralClient diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 3ebd7fa..7847d4e 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -2,40 +2,21 @@ const EventEmitter = require('eventemitter3') const debug = require('debug')('dataparty.match-maker-client') - const dataparty_crypto = require('@dataparty/crypto') const LokiParty = require('../local/loki-party') const PeerParty = require('./peer-party') const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws'){ + constructor(client){ super() - - this.contacts = contacts - this.sessionKey = null - this.identity = identity - this.wsParty = null - this.restParty = null - - if(typeof urlOrParty == 'string'){ - this.restUrl = urlOrParty - this.restParty = null - } else { - this.restParty = urlOrParty - } - - if(typeof wsUrlOrParty == 'string'){ - this.wsUrl = wsUrlOrParty - this.wsParty = null - } else { - this.wsParty = wsUrlOrParty - } + this.client = client this.invitesTx = null this.invitesRx = null @@ -44,80 +25,62 @@ class MatchMakerClient extends EventEmitter { tx: {}, rx: {} } + + this.started = false + + this.client.on('reconnected', this.handleReconnect.bind(this)) + this.client.on('disconnected', this.handleDisconnect.bind(this)) } + async handleReconnect(){ + //if(!this.started){ return } + + debug('handleReconnect') + await this.start() + } + async handleDisconnect(){ + if(!this.started){ return } + + thi.started = false + + debug('handleDisconnect') + this.invitesRx.unsubscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesTx.unsubscribe( this.handleInviteTxMsg.bind(this) ) + + this.invitesRx = null + this.invitesTx = null + + this.emit('disconnected') + } async start(){ - this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) - - if(!this.restParty){ - let config = new MemoryConfig({ - basePath:'match-maker-client', - cloud: { - uri: this.restUrl - } - }) - - this.restParty = new LokiParty({ - path: 'match-maker-client', - dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), - config - }) - - await this.restParty.setIdentity(this.sessionKey) - - debug('starting restParty') - await this.restParty.start() - - if(!this.restParty.comms){ - this.restParty.comms = new Dataparty.Comms.RestComms({ - party:this.restParty, - config: this.restParty.config - }) - - this.restParty.comms.sessionId = this.sessionKey.key.hash - } - await this.announcePublicKeys() - } + if(this.started){ return } - if(!this.wsParty){ - this.wsParty = new PeerParty({ - comms: new WebsocketComms({ - uri: this.wsUrl, - discoverRemoteIdentity: false, - remoteIdentity: await this.restParty.comms.getServiceIdentity(), - session: this.sessionKey.key.hash - }), - config: this.restParty.config - }) + this.started = true + await this.client.start() - await this.wsParty.start() + const party = this.client.socketPeerParty - debug('starting wsParty') - await this.wsParty.start() - debug('waiting for websocket authorization') - await this.wsParty.comms.authorized() + this.invitesRx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/rx', + messageType: 'Object' + }) - this.invitesRx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/rx', - messageType: 'Object' - }) + this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) - this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesTx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/tx', + messageType: 'Object' + }) - this.invitesTx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/tx', - messageType: 'Object' - }) + this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) - this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) - } - + this.emit('connected') } async handleInviteRxMsg( msg ){ @@ -127,8 +90,8 @@ class MatchMakerClient extends EventEmitter { if(!this.pendingInvites.rx[inviteId] && msg.invite.state == 'invited'){ - const from = await this.lookupPublicKey(msg.invite.fromHash) - const to = await this.lookupPublicKey(msg.invite.toHash) + const from = await this.client.lookupPublicKey(msg.invite.fromHash) + const to = await this.client.lookupPublicKey(msg.invite.toHash) let invite = new PeerInvite(msg.invite, to, this, from) @@ -157,93 +120,24 @@ class MatchMakerClient extends EventEmitter { debug('calling onInviteMsg') await pending.onInviteMsg(msg.invite) - + } } + async createInvite(toHashOrIdentity, {type, service, role, session}, info){ - async announcePublicKeys(){ - const announceData = { - annoucement: { - created: Date.now(), - expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now - sessionKey: { - type: this.sessionKey.key.type, - hash: this.sessionKey.key.hash, - public: this.sessionKey.key.public - }, - actorKey: { - type: this.identity.key.type, - hash: this.identity.key.hash, - public: this.identity.key.public - } - }, - trust: { - actorSig: null, - sessionSig: null - } - } - - - const actorSigMsg = await this.identity.sign(announceData.annoucement, true) - const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) - - debug('actorSigMsg', actorSigMsg) - debug('sessionSigMsg', sessionSigMsg) - - announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) - announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) - - debug('announcePublicKeys', announceData) - - const announceResult = await this.restParty.comms.call('key/announce', announceData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: false - }) - } - - - async lookupPublicKey(hash){ - debug('lookupPublicKey - hash:', hash) - - if(hash == this.identity.key.hash){ - return this.identity - } - - if(this.contacts){ - return await this.contacts.lookupPublicKey(hash) - } - - const lookupData = { hash } - - const lookupResult = await this.restParty.comms.call('key/lookup', lookupData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) + const roles = ['client', 'host'] - if(!lookupResult.done){ - return null + if(roles.indexOf(role) == -1){ + throw new Error("Invalid requested role [" + role + "]") } - debug('lookup result -', lookupResult) - - const identity = new dataparty_crypto.Identity({ - key: lookupResult.public_key - }) - - return identity - } - - async createInvite(toHashOrIdentity, {type, service, role, session}, info){ - debug('createInvite') let toIdentity = null if(typeof toHashOrIdentity == 'string'){ - toIdentity = await this.lookupPublicKey(toHashOrIdentity) + toIdentity = await this.client.lookupPublicKey(toHashOrIdentity) } else { toIdentity = toHashOrIdentity } @@ -255,7 +149,7 @@ class MatchMakerClient extends EventEmitter { service: service ? service : '@dataparty/video-chat', role: role ? role : 'client', timestamp: (new Date()).getTime(), - from: this.identity.key.hash, + from: this.client.identity.key.hash, to: toIdentity.key.hash, session: session ? session : Math.random().toString(36).slice(2), info: info ? info : { @@ -264,17 +158,17 @@ class MatchMakerClient extends EventEmitter { } } - const secureInvite = await this.identity.encrypt(invitePayload, toIdentity) + const secureInvite = await this.client.identity.encrypt(invitePayload, toIdentity) debug('secure-invite', secureInvite) const invitePostData = { to: toIdentity.key.hash, - from: this.identity.key.hash, + from: this.client.identity.key.hash, payload: JSON.stringify(secureInvite.toJSON()) } - const inviteResult = await this.restParty.comms.call('invite/create', invitePostData, { + const inviteResult = await this.client.socketPeerParty.comms.call('invite/create', invitePostData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -284,9 +178,9 @@ class MatchMakerClient extends EventEmitter { if(!inviteDoc){ return } - let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.identity) + let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.client.identity, invitePayload) - invite.payload = invitePayload + //invite.payload = invitePayload this.pendingInvites.tx[inviteDoc.$meta.id] = invite @@ -296,16 +190,16 @@ class MatchMakerClient extends EventEmitter { } async lookupInvites({createdAfter, type='to', id, actorHash }){ - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const lookup = { invite: id, - actor: actorHash ? actorHash : this.identity.key.hash, + actor: actorHash ? actorHash : this.client.identity.key.hash, createdAfter, type: !type ? 'to' : type } - const lookupResult = await this.restParty.comms.call('invite/lookup', lookup, { + const lookupResult = await this.client.socketPeerParty.comms.call('invite/lookup', lookup, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -341,8 +235,8 @@ class MatchMakerClient extends EventEmitter { for(let i=0; i < invites.length; i++){ const invite = invites[i] - let to = await this.lookupPublicKey( invite.toHash ) - let from = await this.lookupPublicKey( invite.fromHash ) + let to = await this.client.lookupPublicKey( invite.toHash ) + let from = await this.client.lookupPublicKey( invite.fromHash ) let peerInvite = new PeerInvite( invites[i], to, this, from) @@ -363,14 +257,14 @@ class MatchMakerClient extends EventEmitter { async setInviteState(invite, newState){ debug('setInviteState') - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const inviteState = { invite: invite.inviteDoc.$meta.id, state: newState } - const inviteStateResult = await this.restParty.comms.call('invite/set-state', inviteState, { + const inviteStateResult = await this.client.socketPeerParty.comms.call('invite/set-state', inviteState, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -385,48 +279,6 @@ class MatchMakerClient extends EventEmitter { return inviteStateResult.invite } - async createShortCode(use_limit=3, expiry){ - debug('createShortCode') - - const request = { - use_limit, - expiry: !expiry ? Date.now()+24*60*60*3 : expiry - } - - const result = await this.restParty.comms.call('short-code/create', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('createShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } - - async lookupPublicKeyByShortCode( code ){ - debug('lookupPublicKeyByShortCode') - - const request = { code } - - const result = await this.restParty.comms.call('short-code/lookup', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('lookupPublicKeyByShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } } module.exports = MatchMakerClient diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js new file mode 100644 index 0000000..704f6ab --- /dev/null +++ b/src/party/peer/peer-client.js @@ -0,0 +1,102 @@ +const EphemeralClient = require("./ephemeral-client") + + +class PeerClient extends EphemeralClient { + constructor({model=null, /*hostParty=null, */ contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + + super({identity, contacts, role}) + + this.model = model + this.hostParty = hostParty + this.matchMaker = matchMaker + + this.remoteIdentityHash = remoteIdentityHash + + + this.inviteSettings = { + type: 'webrtc', + service: model ? model.package.name : service, + role: role ? role : 'client', + session: null + } + + this.rtcSettings = this.rtcSettings + + this.peerParty = null + + /*if(role == 'host' && hostParty==null){ + throw + }*/ + } + + get restParty(){ + return this.peerParty + } + + get socketParty(){ + return this.peerParty + } + + async start(mediaSrc){ + // + + if(this.sessionKey){ return } + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + this.inviteSettings.session = this.sessionKey.key.hash + + const role = this.inviteSettings.role + + this.emit('connecting', {time: Date.now()}) + const invite = await this.announcePublicKeys() + + await invite.waitForAccepted() + + this.peerParty = await invite.establish({ + mediaSrc, + role, + hostParty: role == 'host' ? this. this.hostParty : undefined, + model: role == 'host' ? this.hostParty.factory.model : this.model, + rtcSettings: this.rtcSettings + }) + + this.emit('connected', {time: Date.now()}) + + return this.peerParty + } + + async rollSessionKey(){ + debug('rollSessionKey') + this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) + + if(this.peerParty){ + await this.peerParty.stop() + } + + this.sessionKey = null + this.peerParty = null + + await this.start() + } + + async handleClose(){ + // + } + + async doReconnect(){ + // + } + + async announcePublicKeys(){ + const announceData = await this.createSessionAnnoucement() + + let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings, announceData) + + this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) + + return invite + } + +} + +module.exports = PeerClient diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index dbd3f8b..59bfa9f 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -8,6 +8,8 @@ const dataparty_crypto = require('@dataparty/crypto') const PeerParty = require('./peer-party') const RTCSocketComms = require('../../comms/rtc-socket-comms') +const DEFAULT_EXPIRY = 5*60*1000 + const END_STATES = [ 'cancelled', 'rejected', 'expired', 'completed' ] @@ -33,7 +35,7 @@ async function delay(ms){ } class PeerInvite extends EventEmitter { - constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity){ + constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity, payload=null){ super() this.peerParty = null @@ -42,7 +44,7 @@ class PeerInvite extends EventEmitter { this.matchMaker = matchMakerClient this.inviteDoc = inviteDoc this.inviteMsg = null //this.latestDoc = null - this.payload = null + this.payload = payload this.topicSub = null this.topicPub = null @@ -54,6 +56,21 @@ class PeerInvite extends EventEmitter { this.incomingStream = null + this.timeoutTimer = null + + this.role = null + + if(this.payload){ + this._updateRole() + const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() + + const delta = expiry - now + if(delta > 0){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + } + } + /*if(!this.isSender()){ this.inviteDoc. }*/ @@ -66,14 +83,26 @@ class PeerInvite extends EventEmitter { get to(){ return this.toIdentity } get from(){ return this.fromIdentity } + _updateRole(){ + + if(this.isSender()){ + + this.role = this.payload.role + return + + } + + this.role = this.payload.role == 'client' ? 'host' : 'client' + } + isSender(doc){ if(doc){ - if(doc.toHash == matchMaker.identity.key.hash){return false } + if(doc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } - if(this.inviteDoc.toHash == matchMaker.identity.key.hash){return false } + if(this.inviteDoc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } @@ -86,10 +115,10 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept(mediaSrc, config){ + async accept({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity=false}){ debug('accepting invite') - /*if(this.inviteDoc.toHash == matchMaker.wsParty.identity.key.hash){ + /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.fromHash) } else { otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.toHash) @@ -102,13 +131,22 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(JSON.parse(changedInvite.payload)) - let payload = await this.matchMaker.identity.decrypt( + let payload = await this.matchMaker.client.identity.decrypt( msgWorkAround ) this.payload = payload.msg + this._updateRole() + + /*const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() - return await this.establish({mediaSrc, config}) + const delta = expiry - now + if(delta > 0 && this.timeoutTimer != null){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + }*/ + + return await this.establish({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity}) } async reject(){ @@ -120,6 +158,10 @@ class PeerInvite extends EventEmitter { return (this.inviteMsg || this.inviteDoc).state } + async handleTimeout(){ + await this.matchMaker.setInviteState(this, 'expired') + } + async onInviteMsg(inviteMsg){ debug('onInviteMsg', inviteMsg) @@ -158,17 +200,17 @@ class PeerInvite extends EventEmitter { }) } - async establish({mediaSrc, hostParty, config, rtcSettings}){ + async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings, discoverRemoteIdentity=false}){ if(!rtcSettings){ rtcSettings = {} } - let host = this.isSender() + let host = (this.role == 'host') let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from - let party = this.matchMaker.wsParty + let party = this.this.matchMaker.client.socketParty this.topicSub = new party.ROSLIB.Topic({ ros : party.comms.ros, @@ -190,7 +232,7 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(msg.offers[i]) - let offer = await this.matchMaker.identity.decrypt(msgWorkAround) + let offer = await this.matchMaker.client.identity.decrypt(msgWorkAround) if(offer.from.hash != otherIdentity.key.hash){ debug('BAD IDENTITY') @@ -216,6 +258,10 @@ class PeerInvite extends EventEmitter { }*/ this.peerParty = new PeerParty({ + hostParty, + hostRunner, + model: hostParty.factory.model, + config: hostParty.config, comms: new RTCSocketComms({ host: this.isSender(), session: this.payload.session, @@ -230,11 +276,9 @@ class PeerInvite extends EventEmitter { config: DEFAULT_ICE_SERVERS }, trickle: rtcSettings.trickle? rtcSettings.trickle : true, - discoverRemoteIdentity: false, - remoteIdentity: otherIdentity - }), - hostParty: this.isSender() ? hostParty : undefined, - config: config ? config : hostParty.config + discoverRemoteIdentity: discoverRemoteIdentity ? discoverRemoteIdentity : false, + remoteIdentity: discoverRemoteIdentity ? undefined: otherIdentity + }) }) @@ -268,7 +312,7 @@ class PeerInvite extends EventEmitter { debug(' >> offer signal trickle', data) - const secureOffer = await this.matchMaker.identity.encrypt(data, otherIdentity) + const secureOffer = await this.matchMaker.client.identity.encrypt(data, otherIdentity) if(host && !sendFreely){ //console.log('am host') @@ -312,6 +356,7 @@ class PeerInvite extends EventEmitter { } catch (err){ console.log(err) + throw err } } diff --git a/src/party/peer/peer-party.js b/src/party/peer/peer-party.js index 889286d..0502d61 100644 --- a/src/party/peer/peer-party.js +++ b/src/party/peer/peer-party.js @@ -62,6 +62,12 @@ class PeerParty extends IParty { await this.comms.start() } + async stop(){ + if(this.comms){ + await this.comms.stop() + } + } + async handleCall(ask){ debug('handleCall') diff --git a/src/service/endpoint-context.js b/src/service/endpoint-context.js index 80c725a..272a550 100644 --- a/src/service/endpoint-context.js +++ b/src/service/endpoint-context.js @@ -15,7 +15,7 @@ class EndpointContext { * @param {Debug} options.debug Debug constructor (defaults to npm:Debug) * @param {boolean} options.sendFullErrors Enables sending full stack traces to client (defaults to false) */ - constructor({party, endpoint, req, res, input, debug=Debug, sendFullErrors=false}){ + constructor({party, endpoint, runner, req, res, input, debug=Debug, sendFullErrors=false}){ /** * @member module:Service.EndpointContext.debug @@ -27,6 +27,11 @@ class EndpointContext { */ this.endpoint = endpoint + /** + * @member module:Service.EndpointContext.runner + */ + this.runner = runner + /** * @member module:Service.EndpointContext.MiddlewareConfig */ diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js new file mode 100644 index 0000000..26afb71 --- /dev/null +++ b/src/service/endpoints/key-announce.js @@ -0,0 +1,221 @@ +const Joi = require('@hapi/joi') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('dataparty.endpoint.key-announce') + +const {Identity, Message, Routines} = require('@dataparty/crypto') + +//const IEndpoint = require('@dataparty/api/src/service/iendpoint') +const IEndpoint = require('../iendpoint') + +const KeyVerifier = Joi.object().keys({ + id: Joi.string().max(100), + type: Joi.string().max(300).required(), + hash: Joi.string().max(200).required(), + public: { + box: Joi.string().max(60).required(), + sign: Joi.string().max(60).required(), + pqkem: Joi.string().max(8000).required(), + pqsign_ml: Joi.string().max(8000).required(), + pqsign_slh: Joi.string().max(800).required() + } +}) + +module.exports = class KeyAnnounceEndpoint extends IEndpoint { + + static get Name(){ + return 'key/announce' + } + + static get Description(){ + return 'announce a public key' + } + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + validate: Joi.object().keys({ + + annoucement: { + role: Joi.string().valid('guest', 'billing').required(), + created: Joi.number().required(), + expiry: Joi.number().required(), + actorKey: KeyVerifier.required(), + sessionKey: KeyVerifier.required(), + }, + trust:{ + actorSig: Joi.string().required().description('actor signature of the annoucement in base64'), + sessionSig: Joi.string().required().description('session signature of the annoucement in base64') + } + + }).description('key to announce') + }, + post: { + encrypt: true, + validate: Joi.object().keys({ + done: Joi.boolean(), + }).description('public key') + } + } + } + + static async run(ctx, {Package}){ + + ctx.debug('hello key/announce') + + ctx.debug('ip', ctx.req.ip) + ctx.debug('input', ctx.input) + ctx.debug('sender', ctx.senderKey) + + const inputActorKey = { + type: ctx.input.annoucement.actorKey.type, + hash: ctx.input.annoucement.actorKey.hash, + public: ctx.input.annoucement.actorKey.public + } + + const inputSessionKey = { + type: ctx.input.annoucement.sessionKey.type, + hash: ctx.input.annoucement.sessionKey.hash, + public: ctx.input.annoucement.sessionKey.public + } + + + const computedActorHash = await Routines.hashKey( inputActorKey ) + const computedSessionHash = await Routines.hashKey( inputSessionKey ) + + ctx.debug('computed hash -', computedSessionHash) + + // verify keys are self consistent + if( + computedActorHash != inputActorKey.hash || + computedSessionHash != inputSessionKey.hash + ) { + ctx.debug('invalid actor or session key hash') + return {done: false} + } + + + // ensure sender is connected using session key mentioned in annoucement OR this is an internal call + if(computedSessionHash == inputSessionKey.hash && + ( + ctx.req.source == 'INTERNAL' || + //ctx.req.source == 'PeerComms' || + ( + inputSessionKey.public.sign == ctx.senderKey.public.sign && + inputSessionKey.public.box == ctx.senderKey.public.box + ) + ) + ){ + + const actorSigBson = Routines.Utils.base64.decode( ctx.input.trust.actorSig ) + const sessionSigBson = Routines.Utils.base64.decode( ctx.input.trust.sessionSig ) + + const actorSigMsg = new Message({ msg: ctx.input.annoucement }) + const sessionSigMsg = new Message({ msg: ctx.input.annoucement }) + + actorSigMsg.sig = actorSigBson + sessionSigMsg.sig = sessionSigBson + + const actorIdentity = Identity.fromJSON({ + id: '', + key: inputActorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: '', + key: inputSessionKey + }) + + //verify actor & session signature. Require postquantum signing + await actorSigMsg.assertVerified( actorIdentity, true ) + await sessionSigMsg.assertVerified( sessionIdentity, true ) + + // verify actor is allowed + const isAllowed = (await ctx.runner.auth.isAdmin(actorIdentity)) || + (await ctx.runner.auth.isSocketConnectionAllowed(actorIdentity)) + if(!isAllowed){ + ctx.debug('non-allowed user') + return {done: false} + } + + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash').equals(computedSessionHash) + .exec())[0] + + if(!sessionKeyDoc){ + // create session document + + const now = Date.now() + const tomorrow = now + 24*60*60*1000 + + const fiveMinAgo = now - (5*1000*60) + const fiveMinFromNow = now + (5*1000*60) + + // verify start time is valid + if( + ctx.input.annoucement.created < fiveMinAgo || + ctx.input.annoucement.created > fiveMinFromNow + ) { + ctx.debug('invalid start time') + return {done: false} + } + + // verify expiry time is valid + if( + ctx.input.annoucement.expiry < now || + ctx.input.annoucement.expiry > tomorrow + ) { + ctx.debug('invalid expiry time') + return {done: false} + } + + ctx.debug('opening session -', computedSessionHash) + + let sessionDoc = await ctx.party.createDocument('session_key', { + created: now, + expiry: ctx.input.annoucement.expiry, + annoucement: ctx.input.annoucement, + trust: ctx.input.trust + }) + + ctx.debug('session created - ', computedSessionHash) + + } else { + ctx.debug('session key already known - ', computedSessionHash) + return {done: false} + } + + let publicKey = (await ctx.party.find() + .type('public_key') + .where('hash').equals(computedActorHash) + .exec())[0] + + if(!publicKey){ + + ctx.debug('annoucing key -', computedActorHash) + + let keyDoc = await ctx.party.createDocument('public_key', { + created: Date.now(), + role: ctx.input.annoucement.role || 'guest', + owner: computedActorHash, + ...inputActorKey + }) + + ctx.debug('actor annouced key -', computedActorHash) + } else { + ctx.debug('key already known -', computedActorHash) + } + + return {done: true} + + } else { + + ctx.debug('announce ERROR - BAD KEY HASH') + + return {done: false} + } + + } +} diff --git a/src/service/endpoints/service-version.js b/src/service/endpoints/service-version.js index b84c91e..ab5ed11 100644 --- a/src/service/endpoints/service-version.js +++ b/src/service/endpoints/service-version.js @@ -27,7 +27,8 @@ module.exports = class ServiceVersion extends IEndpoint { name: Joi.string(), branch: Joi.string(), version: Joi.string(), - githash: Joi.string() + githash: Joi.string(), + owner: Joi.string(), }) } } diff --git a/src/service/iauth.js b/src/service/iauth.js index c83ef0e..5db6e17 100644 --- a/src/service/iauth.js +++ b/src/service/iauth.js @@ -37,11 +37,16 @@ module.exports = class IAuth { return identity } + async isPeerConnectionAllowed(identity){ + return true + } + async isSocketConnectionAllowed(identity){ //throw new Error('not implemented') return true } + async isInternal(identity){ return false } diff --git a/src/service/index-browser.js b/src/service/index-browser.js index e026bf2..d33a83a 100644 --- a/src/service/index-browser.js +++ b/src/service/index-browser.js @@ -50,4 +50,12 @@ exports.endpoint_paths = { secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), version: Path.join(__dirname, './endpoints/service-version.js') +} + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/service/index.js b/src/service/index.js index bf2ed65..0472ce0 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -24,7 +24,8 @@ exports.MiddlewareRunner= require('./middleware-runner') exports.middleware = { pre: { decrypt: require('./middleware/pre/decrypt'), - validate: require('./middleware/pre/validate') + validate: require('./middleware/pre/validate'), + ephemeral_session: require('./middleware/pre/ephemeral-session.js') }, post: { validate: require('./middleware/post/validate.js'), @@ -35,7 +36,8 @@ exports.middleware = { exports.middleware_paths = { pre: { decrypt: Path.join(__dirname, './middleware/pre/decrypt.js'), - validate: Path.join(__dirname, './middleware/pre/validate.js') + validate: Path.join(__dirname, './middleware/pre/validate.js'), + ephemeral_session: Path.join(__dirname, './middleware/pre/ephemeral-session.js') }, post: { validate: Path.join(__dirname, './middleware/post/validate.js'), @@ -47,12 +49,33 @@ exports.endpoint = { echo: require('./endpoints/echo'), secureecho: require('./endpoints/secure-echo'), identity: require('./endpoints/service-identity'), - version: require('./endpoints/service-version') + version: require('./endpoints/service-version'), + key_announce: require('./endpoints/key-announce') } exports.endpoint_paths = { echo: Path.join(__dirname, './endpoints/echo.js'), secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), - version: Path.join(__dirname, './endpoints/service-version.js') + version: Path.join(__dirname, './endpoints/service-version.js'), + key_announce: Path.join(__dirname, './endpoints/key-announce.js') +} + +exports.schema = { + public_key: require('./schema/public-key'), + session_key: require('./schema/session-key') +} + +exports.schema_paths = { + public_key: Path.join(__dirname, './schema/public-key.js'), + session_key: Path.join(__dirname, './schema/session-key.js') +} + + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/service/iservice.js b/src/service/iservice.js index b9f6c01..2d46caf 100644 --- a/src/service/iservice.js +++ b/src/service/iservice.js @@ -17,7 +17,7 @@ module.exports = class IService { * @param {*} build */ constructor({ - name, version, githash='', branch='' + name, version, githash='', branch='', owner=null }, build){ this.constructors = { @@ -48,11 +48,13 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: null + auth: null, + files: [], + files_root: null } this.compiled = { - package: { name, version, githash, branch }, + package: {owner, name, version, githash, branch }, schemas: { IndexSettings: {}, JSONSchema: [], @@ -70,7 +72,9 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: {} + auth: {}, + files: {}, + //signatures: {} } this.compileSettings = { diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index a0912d8..09dbe3d 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -30,7 +30,14 @@ module.exports = class Encrypt extends IMiddleware { static async run(ctx, {Config}){ if (!Config){ return } - + + if(!ctx.req.source && + ( ctx.req.source == 'PeerComms' || + ctx.req.source == 'INTERNAL' ) + ){ + ctx.setOutput(ctx.output) + return + } const senderStr = JSON.stringify({key: ctx.senderKey}) diff --git a/src/service/middleware/pre/decrypt.js b/src/service/middleware/pre/decrypt.js index 20adf0c..195b521 100644 --- a/src/service/middleware/pre/decrypt.js +++ b/src/service/middleware/pre/decrypt.js @@ -31,8 +31,19 @@ module.exports = class Decrypt extends IMiddleware { if (!Config){ return } + + if(!context.input || !context.input.enc){ - throw new Error('insecure message') + + if(!context.req.source || + ( context.req.source != 'PeerComms' && + context.req.source != 'INTERNAL' ) + ){ + throw new Error('insecure message -' + context.req.source) + } + + context.setInput( context.input ) + return } context.debug('input', context.input, typeof context.input) diff --git a/src/service/middleware/pre/ephemeral-session.js b/src/service/middleware/pre/ephemeral-session.js new file mode 100644 index 0000000..fd099cf --- /dev/null +++ b/src/service/middleware/pre/ephemeral-session.js @@ -0,0 +1,101 @@ +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') + +const IMiddleware = require('../../imiddleware') + +module.exports = class Decrypt extends IMiddleware { + + static get Name(){ + return 'ephemeral_session' + } + + static get Type(){ + return 'pre' + } + + static get Description(){ + return 'Decrypt inbound data' + } + + static get ConfigSchema(){ + return Joi.boolean() + } + + static async start(party){ + + } + + static async run(ctx, {Config}){ + + if (!Config){ return } + + if(!ctx.input_session_id){ + throw new Error('no session id') + } + + ctx.debug('looking up session -', ctx.input_session_id) + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(ctx.input_session_id) + .exec() + )[0] + + if(!sessionKeyDoc){ + throw new Error('invalid session') + } + + ctx.debug('located session - ', ctx.input_session_id) + + const sessionKey = sessionKeyDoc.data.annoucement.sessionKey + const actorKey = sessionKeyDoc.data.annoucement.actorKey + + // ensure sender is connected using session key mentioned in db + if(sessionKey.hash == ctx.input_session_id && + sessionKey.public.sign == ctx.senderKey.public.sign && + sessionKey.public.box == ctx.senderKey.public.box + ){ + + const now = Date.now() + + if( sessionKeyDoc.data.expiry < now ){ + throw new Error('session expired!') + } + + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: actorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: 'ephemeral-session', + key: sessionKey + }) + + ctx.debug('looking up actor - ', actorKey.hash) + + let actorDoc = (await ctx.party.find() + .type('public_key') + .where('hash') + .equals(actorKey.hash) + .exec() + )[0] + + if(!actorDoc){ + throw new Error('failed to find actor') + } + + ctx.setSession( sessionKeyDoc ) + ctx.setIdentity( actorIdentity ) + ctx.setActor( actorDoc ) + + ctx.debug('session ready') + } else { + throw new Error('invalid sender key for this session') + } + + } +} \ No newline at end of file diff --git a/src/service/runner-router.js b/src/service/runner-router.js index c6897b8..24ae4dc 100644 --- a/src/service/runner-router.js +++ b/src/service/runner-router.js @@ -69,7 +69,7 @@ class RunnerRouter { * @returns {module:Service.ServiceRunner} */ getRunnerByHostIdentity(identity){ - const partyId = identity.toString() + const partyId = typeof identity !== 'string' ? identity.key.hash : identity debug('getRunnerByHostIdentity -', partyId) const runner = this.runnersByHost.get(partyId) @@ -83,7 +83,7 @@ class RunnerRouter { */ addRunner({domain, runner}){ - const partyId = runner.party.identity.toString() + const partyId = runner.party.identity.key.hash debug('addRunner - ', partyId, domain) if(!this.runnersByHost.has(partyId)){ diff --git a/src/service/schema/public-key.js b/src/service/schema/public-key.js new file mode 100644 index 0000000..597601b --- /dev/null +++ b/src/service/schema/public-key.js @@ -0,0 +1,47 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + +class PublicKey extends ISchema { + + static get Type () { return 'public_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + + role: String, // [ guest, billing, service, wallet ] + + owner: {required: true, index: true, type: String}, // public key hash + + type: String, + hash: {required: true, index: true, type: String, unique: true}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } + } + + static setupSchema(schema){ + //schema.index({ 'hash': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = PublicKey diff --git a/src/service/schema/session-key.js b/src/service/schema/session-key.js new file mode 100644 index 0000000..a0da93c --- /dev/null +++ b/src/service/schema/session-key.js @@ -0,0 +1,69 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + + +function PublicKeySchema(unique=true){ + return { + type: {required: true, type: String}, + hash: {required: true, index: true, type: String, unique}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } +} + + +class SessionKey extends ISchema { + + static get Type () { return 'session_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + index: true, + required: true + }, + annoucement: { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + required: true + }, + sessionKey: PublicKeySchema(true), + actorKey: PublicKeySchema(false) + }, + trust: { + actorSig: {required: true, type: String}, //! base64 of BSON signature + sessionSig: {required: true, type: String} //! base64 of BSON signature + } + } + } + + static setupSchema(schema){ + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = SessionKey \ No newline at end of file diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 614afd9..0598e4c 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -7,6 +7,18 @@ const gitRepoInfo = require('git-repo-info') const BouncerDb = require('@dataparty/bouncer-db') const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') +const zlib = require('zlib') + +const safeStringify = require('fast-safe-stringify') + +const dataparty_crypto = require('@dataparty/crypto') + +const { + globSync +} = require('glob') +const { isArray } = require('lodash') + +const tar = require('tar') //const IService = require('../iservice') @@ -27,7 +39,7 @@ module.exports = class ServiceBuilder { * @param {boolean} writeFile When true, files will be written. Defaults to `true` * @returns */ - async compile(outputPath, writeFile=true){ + async compile(outputPath, writeFile=true, owner){ if(!outputPath){ throw new Error('no output path') @@ -40,7 +52,7 @@ module.exports = class ServiceBuilder { debug('compiling sources',this.service.sources) - await Promise.all([ + let results = await Promise.all([ this.compileMiddleware('pre'), this.compileMiddleware('post'), this.compileList('documents'), @@ -48,25 +60,57 @@ module.exports = class ServiceBuilder { this.compileList('tasks'), this.compileList('topics'), this.compileFile('auth'), - this.compileSchemas() + this.compileSchemas(), + this.compressFiles(outputPath, writeFile) ]) + + this.service.compiled.middleware_order = this.service.middleware_order this.service.compiled.compileSettings = this.service.compileSettings + if(owner){ + this.service.compiled.package.owner = owner.key.hash + const ownerSig = await owner.sign(this.service.compiled, true) + + console.log('sign keys', Object.keys(this.service.compiled)) + console.log(this.service.compiled.signature) + + this.service.compiled.signatures = { + [owner.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + } + + let files = [] + if(writeFile){ - const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-service.json' + const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.service.venue.json' fs.writeFileSync(buildOutput, JSON.stringify(this.service.compiled, null,2)) - const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-schema.json' + const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.schema.venue.json' fs.writeFileSync(schemaOutput, JSON.stringify({ package: this.service.compiled.package, ...this.service.compiled.schemas }, null, 2)) + + // Gzip compression (most common for HTTP) + const compressed = zlib.gzipSync(JSON.stringify(this.service.compiled, null,2)); + + // Brotli compression (better ratio, Node.js 10.5.0+) + const compressedBrotli = zlib.brotliCompressSync(JSON.stringify(this.service.compiled, null,2)); + + console.log('Original:', JSON.stringify(this.service.compiled, null,2).length, 'bytes'); + console.log('Gzip:', compressed.length, 'bytes') + console.log('Brotli:', compressedBrotli.length, 'bytes') + + let tarFile = results[ results.length - 1 ] + files.push(buildOutput) + files.push(schemaOutput) + if(tarFile){files.push(tarFile)} } - return this.service.compiled + return {build: this.service.compiled, files} } @@ -142,24 +186,33 @@ module.exports = class ServiceBuilder { this.service.compiled.schemas.Permissions[model.Type] = await model.permissions() this.service.compiled.schemas.JSONSchema.push(jsonSchema) + const safePaths = JSON.parse(safeStringify(schema.paths)) + + //debug(schema.paths) debug('\t','type',model.Type) let indexed = JSONPath({ path: '$..options.index', - json: schema.paths, + json: safePaths, resultType: 'pointer' - }).map(p=>{return p.split('/')[1]}) + }).map(p=>{ + + debug('\t\t','indexed p',p) + return p.replace('/options/index', '').replace('/','') + }) + //return p.split('.')[1]}) debug('\t\tindexed', indexed) let unique = JSONPath({ path: '$..options.unique', - json: schema.paths, + json: safePaths, resultType: 'pointer' }).map(p=>{ - debug(typeof p) + debug(typeof p, 'unique', p) if(typeof p == 'string'){ - return p.split('/')[1] + let filteredP = p.replace('/options/unique', '').replace('/','') + return filteredP } return p @@ -315,4 +368,70 @@ module.exports = class ServiceBuilder { this.service.sources.auth = auth_path this.service.constructors.auth = TopicClass } + + addFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.service.files){ + this.service.sources.files = result + } else { + this.service.sources.files = this.service.sources.files.concat(result) + } + + this.service.sources.files_root = root + + debug('addFiles',result) + + } + + async compressFiles(outputPath, writeFile){ + + if(!this.service.sources.files){ return } + + let fileMap={} + + let files = this.service.sources.files.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + if(!files || files.length < 1){ return } + + const tarFileName = this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) + + await tar.create({ + cwd: this.service.sources.files_root, + gzip: true, + file: tarPath + }, this.service.sources.files) + + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + this.service.compiled.files = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } + + return tarPath + } } \ No newline at end of file diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js new file mode 100644 index 0000000..03c37a1 --- /dev/null +++ b/src/service/service-host-peer.js @@ -0,0 +1,77 @@ +const debug = require('debug')('dataparty.service.host-peer') + +class ServiceHostPeer { + + constructor({ + runner, + matchMaker, + mediaSrc, + discoverRemoteIdentity = false + }){ + this.runner = runner + this.matchMaker = matchMaker + this.mediaSrc = mediaSrc + this.discoverRemoteIdentity = discoverRemoteIdentity + } + + async start(){ + // + + this.matchMaker.on('invited', this.onInvite.bind(this)) + } + + async onInvite(invite){ + + + // Filter out none host mode requests + if(invite.role !== 'host'){ + debug('FAIL - unexpected role[', invite.role, '] we expecte to be host') + await invite.reject() + return + } + + let hostRunner = this.runner.party ? this.runner : this.runner.getRunnerByHostIdentity(invite.to) + + // Make sure we know the requested party & runner + if(!hostRunner){ + debug('FAIL - requested party not available', invite.to) + await invite.reject() + return + } + + //! Check if party wants to allow/deny invite sender + const isAllowed = (await hostRunner.auth.isAdmin(invite.from)) || + (await hostRunner.auth.isPeerConnectionAllowed(invite.from)) || + (await hostRunner.auth.isSocketConnectionAllowed(invite.from)) + if(!isAllowed){ + debug('NOT ALLOWED - user is not allowed', invite.from) + await invite.reject() + return + } + + + //! Announce the session key + const annoucement = invite.payload.info + const result = await hostRunner.internalRequest('key/announce', annoucement) + + debug('annoucement result', result) + + if(!result || !result.done){ + + debug('user session not allowed') + + return + } + + let hostParty = hostRunner.party + + const peerParty = await invite.accept({ + mediaSrc: this.mediaSrc, + hostParty, + hostRunner, + discoverRemoteIdentity: this.discoverRemoteIdentity //! todo - this is probably something a party config could reasonably over ride + }) + } +} + +module.exports = ServiceHostPeer \ No newline at end of file diff --git a/src/service/service-host.js b/src/service/service-host.js index 6fd1c68..85bb43e 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -58,8 +58,9 @@ class ServiceHost { i2pSamHost = '127.0.0.1', i2pSamPort = 7656, i2pKey = null, - i2pForwardHost = 'localhost', + i2pForwardHost = '127.0.0.1', i2pForwardPort = null, + i2pOptions = null, wsEnabled = true, wsPort = null, wsUpgradePath = '/ws', @@ -113,8 +114,8 @@ class ServiceHost { if(debug.enabled){ this.apiApp.use(morgan('combined')) } this.apiApp.use(bodyParser.urlencoded({ extended: true })) - this.apiApp.use(bodyParser.json()) - this.apiApp.use(bodyParser.raw()) + this.apiApp.use(bodyParser.json({limit:'10MB'})) + this.apiApp.use(bodyParser.raw({limit:'10MB'})) this.apiApp.set('trust proxy', trust_proxy) @@ -141,11 +142,17 @@ class ServiceHost { host: i2pSamHost, portTCP: i2pSamPort, publicKey: reach(i2pKey, 'publicKey'), - privateKey: reach(i2pKey, 'privateKey') + privateKey: reach(i2pKey, 'privateKey'), + versionMin: '3.1', + versionMax: '3.3' }, forward: { + silent: true, host: i2pForwardHost ? i2pForwardHost : this.apiServerUri.hostname, port: i2pForwardPort ? i2pForwardPort : parseInt( this.apiServerUri.port ) + }, + session: { + options: i2pOptions } } } @@ -246,29 +253,32 @@ class ServiceHost { if(this.i2pEnabled && this.i2p == null){ debug('starting i2p forward', this.i2pSettings) - const SAM = require('@diva.exchange/i2p-sam') - this.i2p = await SAM.createForward(this.i2pSettings) - this.i2pUri = this.i2p.getB32Address() - this.i2pSettings.privateKey = null // clear no longer needed + async function setup_i2p(){ + const SAM = require('@diva.exchange/i2p-sam') + this.i2p = await SAM.createForward(this.i2pSettings) + this.i2pUri = this.i2p.getB32Address() + this.i2pSettings.sam.privateKey = null // clear no longer needed + this.i2p.on('error', this.reportI2pError.bind(this)) - this.i2p.on('error', this.reportI2pError.bind(this)) + this.i2p.on('close', ()=>{ + debug('i2p closed') + }) - this.i2p.on('close', ()=>{ - debug('i2p closed') - }) + this.i2p.on('data', (data)=>{ + debug('i2p data') + debug(data.toString()) + }) - this.i2p.on('data', (data)=>{ - debug('i2p data') - debug(data.toString()) - }) + debug('i2p started') + debug('\t', 'address', this.i2pUri) + debug('\t', 'key', this.i2p.getPublicKey()) + } - debug('i2p started') - debug('\t', 'address', this.i2pUri) - debug('\t', 'key', this.i2p.getPublicKey()) + setup_i2p.bind(this)() } if(this.mdnsEnabled && this.apiServer && this.apiServerUri.protocol != 'file:'){ @@ -307,7 +317,7 @@ class ServiceHost { this.apiApp.use((err, req, res, _next) => { console.log('Error handler', err) if (err instanceof IpDeniedError) { - //res.status(401) + res.status(401) } else { res.status(err.status || 500) } diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index 5df899c..7d93b19 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -5,6 +5,7 @@ const Debug = require('debug') const debug = Debug('dataparty.service.runner-node') const EndpointContext = require('./endpoint-context') const DeltaTime = require('../utils/delta-time') +const HttpMocks = require('node-mocks-http') const Router = require('origin-router').Router const Runner = require('@dataparty/tasker').Runner @@ -110,13 +111,13 @@ class ServiceRunnerNode { let AuthClass = null - if(!this.useNative){ + if(!this.useNative && Hoek.reach(this.service, `compiled.auth`)){ var self={} const build = Hoek.reach(this.service, `compiled.auth`) eval(build.code/*, build.map*/) AuthClass = self.Lib } - else{ + else if(this.service.constructors.auth){ AuthClass = this.service.constructors.auth } @@ -446,6 +447,38 @@ class ServiceRunnerNode { } } + async internalRequest(endpoint, data){ + let bodyValue = data + + const req = HttpMocks.createRequest({ + method: 'GET', + url: '/'+endpoint, + body: bodyValue + }) + + const res = HttpMocks.createResponse() + + debug('\tthe request', req) + + debug('req ip type', typeof req.ip) + + const route = this.router.get(endpoint) + + debug('route',route) + + req.runner = this + req.source = 'INTERNAL' + + debug('call route', await route._events.route({ + method: req.method, + pathname: req.url, + request: req, + response: res + })) + + return {result: res._getData() } + } + async getTopic(path){ debug('looking up topic', path) @@ -516,6 +549,7 @@ class ServiceRunnerNode { req: event.request, res: event.response, endpoint, party: this.party, + runner: this, input: event.request.body, debug: Debug, sendFullErrors: this.sendFullErrors diff --git a/src/service/tasks/cleanup-ephemeral-sessions.js b/src/service/tasks/cleanup-ephemeral-sessions.js new file mode 100644 index 0000000..18cb7d0 --- /dev/null +++ b/src/service/tasks/cleanup-ephemeral-sessions.js @@ -0,0 +1,86 @@ +const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') + +//const ITask = require('@dataparty/api/src/service/itask') +const ITask = require('../../service/itask') + +class CleanupEphemeralSessionsTask extends ITask { + + constructor(options){ + super({ + name: CleanupEphemeralSessionsTask.name, + background: CleanupEphemeralSessionsTask.Config.background, + ...options + }) + + debug('new') + + this.duration = Math.round(1000*60*15) + this.timeout = null + } + + static get Config(){ + return { + background: true, + autostart: true + } + } + + async exec(){ + + this.setTimer() + + return this.detach() + } + + + async lookupSessions(){ + + let now = Date.now() + + return (await this.context.party.find() + .type('session_key') + .where('expiry').lt(now) + .exec()) + } + + setTimer(){ + this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) + } + + async onTimeout(){ + this.timeout = null + + debug('cleanup ephemeral sessions task') + + try{ + let sessions = await this.lookupSessions() + + if(sessions && sessions.length > 0){ + debug('expired sessions ', sessions.length) + let list = sessions.map(i=>{return i.data}) + await this.context.party.remove(...list) + } + } catch (err){ + debug(err) + } + + this.setTimer() + } + + stop(){ + if(this.timeout !== null){ + clearTimeout(this.timeout) + this.timeout = null + } + } + + static get Name(){ + return 'cleanup-ephemeral-sessions' + } + + static get Description(){ + return 'Cleanup Ephemeral sessions' + } +} + +module.exports = CleanupEphemeralSessionsTask diff --git a/src/venue/auth.js b/src/venue/auth.js index c2ff01c..7b8074c 100644 --- a/src/venue/auth.js +++ b/src/venue/auth.js @@ -1,6 +1,7 @@ const debug = require('debug')('dataparty.auth.venue-auth') const IAuth = require('../service/iauth') +const {Identity} = require('@dataparty/crypto') module.exports = class IAuth { @@ -36,12 +37,28 @@ module.exports = class IAuth { } async lookupIdentity(identity){ + + let sessionKeyDoc = (await this.context.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(identity.key.hash) + .exec() + )[0] + + if(sessionKeyDoc){ + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: sessionKeyDoc.data.annoucement.actorKey + }) + + return actorIdentity + } + return identity } async isSocketConnectionAllowed(identity){ - //throw new Error('not implemented') - return true + return await this.isAdmin(identity) } async isInternal(identity){ @@ -49,7 +66,16 @@ module.exports = class IAuth { } async isAdmin(identity){ - return false + + // verify key-hash is an admin + const admins = (await this.context.party.config.read('admins')) || [] + + if(admins.indexOf(identity.key.hash) == -1){ + debug('non-admin user', identity.key.hash) + return false + } + + return true } async canReadDb(identity){ diff --git a/src/venue/bin/add-admin.js b/src/venue/bin/add-admin.js new file mode 100755 index 0000000..f09375e --- /dev/null +++ b/src/venue/bin/add-admin.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +const Dataparty = require('../../index') + + +async function main(){ + + const path = '/data/dataparty/venue-service' + + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + await config.start() + + console.log(process.argv) + + let admins = (await config.read('admins')) || [] + + const newAdmin = process.argv[2] + + console.log(await config.readAll()) + console.log(admins) + + if(admins.indexOf(newAdmin) != -1){ return } + + admins.push(newAdmin) + + await config.write('admins', admins) + + console.log('admin added -', newAdmin) + +} + + +main().catch(err=>{ + console.error(err) +}).finally(()=>{ + process.exit() +}) diff --git a/src/venue/bin/chill.js b/src/venue/bin/chill.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js new file mode 100644 index 0000000..ee297ba --- /dev/null +++ b/src/venue/bin/commands/pkg-build.js @@ -0,0 +1,170 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.package-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') +const findUp = require('find-up-json').default + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'package name' + }, + version: { + description: 'package version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false + } +} + + +class VenuePackageBuild extends CmdTree.Command { + constructor(context){ + super({...VenuePackageBuild.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'package build' + } + + static get Definition(){ + return { + usage: `venue package build [service-code.js]`, + description: 'Build a package', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply service code') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const serviceClassPath = Path.resolve(parsed._[2]) + + let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content + + const ServiceClass = require( serviceClassPath ) + + const service = new ServiceClass({ + name: parsed.name ? parsed.name : pkgJson.name, + version: parsed.version ? parsed.version : pkgJson.version, + }) + + await mkdirp(parsed.output) + + const builder = new Dataparty.ServiceBuilder(service) + const build = await builder.compile(parsed.output, true, key) + + + if(parsed.deploy && parsed.remote){ + console.log('uploading...') + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + if(!remote){ + throw 'invalid remote ['+parsed.remote+']' + } + + let staticTar = undefined + + if(build.files.length == 3){ + staticTar = fs.readFileSync(build.files[ build.files.length - 1 ]) + } + + await this.pushPackage(key, remote, build.build, staticTar) + } + + return {files: build.files} + } + + + async pushPackage(devId, remote, build, staticTar=null){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } +} + +module.exports = VenuePackageBuild + + diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js new file mode 100644 index 0000000..0a9d8eb --- /dev/null +++ b/src/venue/bin/commands/project-build.js @@ -0,0 +1,283 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.project-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const { + globSync +} = require('glob') +const tar = require('tar') + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') +const Joi = require('joi') + +const {Routines} = dataparty_crypto + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'project name' + }, + version: { + description: 'project version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false + } +} + + +async function compressFiles(projectName, root, fileList, outputPath, writeFile){ + + if(!fileList){ return } + + let fileMap={} + + let files = fileList.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + if(!files || files.length < 1){ return } + + const tarFileName = projectName.replace('/', '-')+'.project.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) + + await tar.create({ + cwd: root, + gzip: true, + file: tarPath + }, fileList) + + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + const fileInfo = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } + + return {tarPath, files: fileInfo} +} + + +class VenueProjectBuild extends CmdTree.Command { + constructor(context){ + super({...VenueProjectBuild.Definition, context}) + debug('constructor') + + this.project = {} + this.project_sources = [] + } + + static get Command(){ + return 'project build' + } + + static get Definition(){ + return { + usage: `venue project build [project.json]`, + description: 'Build a project', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply project json/js') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const projectJsonPath = Path.resolve(parsed._[2]) + + /*let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content*/ + + let projectJson = require( projectJsonPath ) + + + const remoteName = parsed.remote || projectJson.venue + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + if(!remote){ + throw new CmdTree.Error.UsageError('Invalid remote ['+remoteName+']') + } + + const project = { + owner: key.key.hash, + //created: Date.now(), + + name: parsed.name ? parsed.name : projectJson.name, + version: parsed.version ? parsed.version : projectJson.version, + + venue: remote.identity.key.hash, + domain: projectJson.domain, + + i2p: projectJson.i2p, + party: projectJson.party, + routes: projectJson.routes, + files: projectJson.files + + } + + await mkdirp(parsed.output) + + const buildOutput = parsed.output+'/'+ project.name.replace('/', '-') +'.project.venue.json' + + + let prjFiles = [] + prjFiles.push(buildOutput) + + if(project.files){ + this.addProjectFiles( + Path.dirname(projectJsonPath), + projectJson.files, + { nodir: true, follow: true } + ) + + const {tarPath, files} = await compressFiles(project.name,Path.dirname(projectJsonPath), this.project_sources.files, parsed.output, true) + + project.files = files + + if(tarPath){prjFiles.push(tarPath)} + + + } + + const ownerSig = await key.sign( project, true ) + + project.signatures = { + [key.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + + fs.writeFileSync(buildOutput, JSON.stringify(project, null,2)) + + let staticTar = undefined + + if(prjFiles.length == 2){ + staticTar = fs.readFileSync(prjFiles[ prjFiles.length - 1 ]) + } + + await this.pushProject(key, remote, project, staticTar) + + return {files: prjFiles, project} + } + + + addProjectFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.project_sources.files){ + this.project_sources.files = result + } else { + this.project_sources.files = this.project_sources.files.concat(result) + } + + this.project_sources.files_root = root + + debug('addFiles',result) + + } + + async pushProject(devId, remote, build, staticTar=null){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-project', {project:build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } +} + +module.exports = VenueProjectBuild + + diff --git a/src/venue/bin/commands/venue-identity-gen.js b/src/venue/bin/commands/venue-identity-gen.js new file mode 100644 index 0000000..6cb6d2b --- /dev/null +++ b/src/venue/bin/commands/venue-identity-gen.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-gen') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityGen extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityGen.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity gen' + } + + static get Definition(){ + return { + usage: `venue identity gen [name]`, + description: 'Create an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + if(await this.context.secureConfig.read('identity.'+keyName+'.phrase') && !parsed.force){ + throw new CmdTree.Error.UsageError('Key already exists!') + } + + const phrase = await dataparty_crypto.Routines.generateMnemonic() + + const password = parsed.nopassword ? null : await this.context.collectPassword() + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + await this.context.secureConfig.write('identity.'+keyName+'.phrase', phrase) + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityGen + + diff --git a/src/venue/bin/commands/venue-identity-list.js b/src/venue/bin/commands/venue-identity-list.js new file mode 100644 index 0000000..171fdf3 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-list.js @@ -0,0 +1,56 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueIdentityList extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity list' + } + + static get Definition(){ + return { + usage: `venue identity list [name]`, + description: 'List identity nicknames', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('identity')) + + return {names} + } +} + +module.exports = VenueIdentityList + + diff --git a/src/venue/bin/commands/venue-identity-show.js b/src/venue/bin/commands/venue-identity-show.js new file mode 100644 index 0000000..e289c74 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-show.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityShow extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity show' + } + + static get Definition(){ + return { + usage: `venue identity show [name]`, + description: 'Show an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exists!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityShow + + diff --git a/src/venue/bin/commands/venue-remote-add.js b/src/venue/bin/commands/venue-remote-add.js new file mode 100644 index 0000000..96a8093 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-add.js @@ -0,0 +1,94 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteAdd extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteAdd.Definition, context}) + } + + static get Command(){ + return 'remote add' + } + + static get Definition(){ + return { + usage: `venue remote add [name]`, + description: 'Add remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + if(await this.context.secureConfig.read('remote.'+remoteName) && !parsed.force){ + throw new CmdTree.Error.UsageError('Remote already exists!') + } + + const identityUrl = parsed.url+'/identity' + const versionUrl = parsed.url+'/version' + + const identity = await Dataparty.Comms.RestComms.HttpGet(identityUrl) + const version = await Dataparty.Comms.RestComms.HttpGet(versionUrl) + + const remote = { + url: parsed.url, + ws: parsed.ws, + i2p: parsed.i2p, + identity, version + } + + await this.context.secureConfig.write('remote.'+remoteName, remote) + + return { remote } + } +} + +module.exports = VenueRemoteAdd diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js new file mode 100644 index 0000000..bae2595 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-check.js @@ -0,0 +1,127 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-check') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + } +} + + +class VenueRemoteCheck extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteCheck.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote check' + } + + static get Definition(){ + return { + usage: `venue remote check`, + description: 'Check a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + + client.on('connecting',(info)=>{ + console.log('connecting', info) + }) + + client.on('connected',(info)=>{ + console.log('connected', info) + }) + + client.on('disconnected',(info)=>{ + console.log('disconnected', info) + }) + + client.on('reconnected',(info)=>{ + console.log('reconnected', info) + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + + await client.start() + console.log('client started') + + this.context.exiting = false + + return {remote, client} + } +} + +module.exports = VenueRemoteCheck + + diff --git a/src/venue/bin/commands/venue-remote-list.js b/src/venue/bin/commands/venue-remote-list.js new file mode 100644 index 0000000..ed9b1fe --- /dev/null +++ b/src/venue/bin/commands/venue-remote-list.js @@ -0,0 +1,71 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteList extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteList.Definition, context}) + } + + static get Command(){ + return 'remote list' + } + + static get Definition(){ + return { + usage: `venue remote list [name]`, + description: 'List remote parties', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('remote')) + + return { names } + } +} + +module.exports = VenueRemoteList diff --git a/src/venue/bin/commands/venue-remote-repl.js b/src/venue/bin/commands/venue-remote-repl.js new file mode 100644 index 0000000..118fd2d --- /dev/null +++ b/src/venue/bin/commands/venue-remote-repl.js @@ -0,0 +1,138 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-repl') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const repl = require("repl"); + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'admin identity', + require: true + } +} + + +class VenueRemoteRepl extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteRepl.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote repl' + } + + static get Definition(){ + return { + usage: `venue remote repl`, + description: 'Run a repl with access to a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + + client.on('connecting',(info)=>{ + console.log('connecting', info) + }) + + client.on('connected',(info)=>{ + console.log('connected', info) + }) + + client.on('disconnected',(info)=>{ + console.log('disconnected', info) + }) + + client.on('reconnected',(info)=>{ + console.log('reconnected', info) + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + + await client.start() + console.log('client started') + + + + const replSrv = repl.start({ + prompt: parsed.identity +'@'+ parsed.remote+'> ' + }) + + replSrv.context.remote = remote + replSrv.context.client = client + + + this.context.exiting = false + + return // {remote, client, replSrv} + } +} + +module.exports = VenueRemoteRepl + + diff --git a/src/venue/bin/commands/venue-remote-show.js b/src/venue/bin/commands/venue-remote-show.js new file mode 100644 index 0000000..c8f1e63 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-show.js @@ -0,0 +1,66 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueRemoteShow extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote show' + } + + static get Definition(){ + return { + usage: `venue remote show [name]`, + description: 'Show a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + + return {remote} + } +} + +module.exports = VenueRemoteShow + + diff --git a/src/venue/bin/commands/venued-admin-add.js b/src/venue/bin/commands/venued-admin-add.js new file mode 100644 index 0000000..209e6fe --- /dev/null +++ b/src/venue/bin/commands/venued-admin-add.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminAdd extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminAdd.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin add' + } + + static get Definition(){ + return { + usage: `venued admin add [key-hash]`, + description: 'Add admin key', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + + let admins = (await config.read('admins')) || [] + + + let newAdmins = [] + + + for(let admin of parsed._.slice(2)){ + if(admins.indexOf(admin) != -1){ continue } + + newAdmins.push(admin) + } + + admins = admins.concat(newAdmins) + + await config.write('admins', admins) + await config.save() + + //console.log('admin added -', newAdmins) + + return {newAdmins} + } +} + +module.exports = VenuedAdminAdd \ No newline at end of file diff --git a/src/venue/bin/commands/venued-admin-list.js b/src/venue/bin/commands/venued-admin-list.js new file mode 100644 index 0000000..da0555e --- /dev/null +++ b/src/venue/bin/commands/venued-admin-list.js @@ -0,0 +1,69 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminList extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin list' + } + + static get Definition(){ + return { + usage: `venued admin list [key-hash]`, + description: 'List admin keys', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + let config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + let admins = (await config.read('admins')) || [] + + config=null + + return {admins} + } +} + +module.exports = VenuedAdminList \ No newline at end of file diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js new file mode 100644 index 0000000..81200cb --- /dev/null +++ b/src/venue/bin/commands/venued-host.js @@ -0,0 +1,406 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.host') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const zlib = require('zlib') +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const {Routines} = require('@dataparty/crypto') +const express = require('express') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + }, + l: { + alias: 'listen', + description: 'listen uri this can be an https:// with any local IP or file:/// to use a Unix socket', + default: 'https://0.0.0.0:3000' + }, + k: { + alias: 'ssl-key', + description: 'SSL key in pem format', + default: Path.join(DEFAULT_PATH, 'key.pem') + }, + c: { + alias: 'ssl-cert', + description: 'SSL Certificate in pem format', + default: Path.join(DEFAULT_PATH, 'cert.pem') + }, + A: { + alias: 'allow-ip', + description: 'IP to add to allow list', + multiple: true + }, + 'db-type': { + description: 'Type of database', + valid: ['tingo', 'loki', 'mongo', 'peer'], + default: 'tingo' + }, + 'db-uri': { + description: 'url to connect to database', + default: Path.join(DEFAULT_PATH, 'db/') + }, + 'db-peer': { + description: 'Peer database identity hash' + }, + 'service-code': { + description: 'path to service implementation', + default: Path.join(__dirname, '../../venue-service.js') + }, + 'service-build': { + description: 'path to service build', + default: Path.join(__dirname, '../../dataparty/@dataparty-venue.service.venue.json') + }, + 'full-errors': { + description: 'full server side error messages sent to client', + type:'boolean', + default: false + }, + i2p: { + description: 'Enable i2p hosting', + type: 'boolean', + default: false + }, + 'i2p-host':{ + default: '127.0.0.1' + }, + 'i2p-port': { + default: 7656 + }, + 'trust-proxy': { + type: 'boolean', + default: false + } +} + +function getPartyByType(type){ + if(type == 'tingo'){ + return Dataparty.TingoParty + } +} + +class VenuedHost extends CmdTree.Command { + constructor(context){ + super({...VenuedHost.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'host' + } + + static get Definition(){ + return { + usage: `venued host [options]`, + description: 'Start the hosting venued', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + /*if (!parsed.name){ + throw new CmdTree.Error.UsageError('no name provided') + }*/ + + + const ServiceCode = require(parsed['service-code']) + const ServiceBuild = require(parsed['service-build']) + const ServiceSchema = { + package: ServiceBuild.package, + ...ServiceBuild.schemas + } + + const PARTY = getPartyByType(parsed['db-type']) + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + //if(parsed['db-uri'][0] == '/'){ + config.touchDir( 'db' ) + //} + + const party = new PARTY({ + path: parsed['db-uri'], + model: ServiceSchema, + config: config, + noCache: false + }) + + const service = new ServiceCode( ServiceSchema.package, ServiceBuild ) + + debug('loaded service') + + debug('party db location', parsed['db-uri']) + + const CustomIpFilter = { + options: { + mode: 'allow', + //trustProxy: true + }, + ips: [ + '173.245.48.0/20', + '103.21.244.0/22', + '103.22.200.0/22', + '103.31.4.0/22', + '141.101.64.0/18', + '108.162.192.0/18', + '190.93.240.0/20', + '188.114.96.0/20', + '197.234.240.0/22', + '198.41.128.0/17', + '162.158.0.0/15', + '104.16.0.0/13', + '104.24.0.0/14', + '172.64.0.0/13', + '131.0.72.0/22', + '2400:cb00::/32', + '2606:4700::/32', + '2803:f800::/32', + '2405:b500::/32', + '2405:8100::/32', + '2a06:98c0::/29', + '2c0f:f248::/32', + '10.115.68.55/32' //! + ] + } + + if(parsed['allow-ip']){ + for(let ip of parsed['allow-ip']){ + CustomIpFilter.ips.push(ip) + } + } + + + if(!fs.existsSync(parsed['ssl-key'])){ + execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', + {cwd: parsed.path} + ) + } + + const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') + const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') + + + const runner = new Dataparty.ServiceRunnerNode({ + party, service, + sendFullErrors: parsed['full-errors'], + useNative: false, + prefix: 'venue/' + }) + + + let runnerRouter = new Dataparty.RunnerRouter(runner) + + + + + if(parsed.i2p && !await config.read('i2p.sam')){ + debug('i2p - creating key') + + const SAM = require('@diva.exchange/i2p-sam') + + const i2pSettings = { + sam: { + host: parsed['i2p-host'], + portTCP: parsed['i2p-port'], + }, + session: { + options: 'i2cp.leaseSetEncType=6,4' + //options: 'i2cp.leaseSetEncType=4' + } + } + + let i2p = await SAM.createLocalDestination(i2pSettings) + + + await Promise.all([ + config.write('i2p.address', i2p.address), + config.write('i2p.sam.publicKey', i2p.public), + config.write('i2p.sam.privateKey', i2p.private), + ]) + + await config.save() + } + + const host = new Dataparty.ServiceHost({ + runner: runnerRouter, + trust_proxy: parsed['trust-proxy/'], + wsEnabled: true, + ssl_key, ssl_cert, + listenUri: parsed.listen, + staticPath: Path.join(__dirname,'../../public'), + staticPrefix: '/venue/', + ipFilter: CustomIpFilter, + i2pEnabled: parsed.i2p, + i2pSamHost: parsed['i2p-host'], + i2pSamPort: parsed['i2p-port'], + i2pForwardHost: '127.0.0.1', + i2pForwardPort: '3000', + i2pOptions: 'i2cp.leaseSetEncType=6,4', + //i2pOptions: 'i2cp.leaseSetEncType=4', + i2pKey: await config.read('i2p.sam') + }) + + await party.start() + await runner.start() + await host.start() + + debug('started') + console.log('partying') + console.log('\t', parsed.listen) + + const i2pAddress = await config.read('i2p.address') + if(i2pAddress){ + console.log('\t', i2pAddress) + } + + /** + * config section + * + * projects: { + * [name]: latest-hash + * } + */ + + const projects = await config.read('projects') + + if(projects){ + for(let name in projects){ + + let projectParties = {} + + //! todo - load parties and put them in projectParties map + + const hash = projects[name] + console.log('\tloading project', name, hash) + + const project = (await party.find() + .type('venue_project') + .where('hash').equals(hash).exec())[0] + + const workspace = project.data.workspace + + for(let route of project.data.project.routes){ + console.log('route', route.package.name) + let pkgDoc = (await party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + const serviceFile = JSON.parse( + zlib.brotliDecompressSync( + Routines.Utils.base64.decode( compressedBuild ) + ) + ) + + console.log('decompressed', serviceFile.package) + + let serviceParty = null; + + + if(route.party == 'SYSTEM'){ + serviceParty = party + } else if( projectParties[route.party] ){ + serviceParty = projectParties[route.party] + } + + serviceParty.topics = new Dataparty.LocalTopicHost() + + debug('loading service') + const service = new Dataparty.IService(serviceFile.package, serviceFile) + debug('loaded service') + + let projectRunner = new Dataparty.ServiceRunnerNode({ + party: serviceParty, service, + sendFullErrors: route.settings.sendFullErrors, + useNative: route.settings.useNative, + prefix: route.prefix + }) + + if(route.party == 'SYSTEM'){ + //projectRunner.router = runner.router + } + + //await serviceParty.start() + + await projectRunner.start() + + console.log('workspace', workspace) + + const projectStaticPath = Path.join(workspace, 'public') + + let handler = (req,res)=>{ + + let staticHandler = express.static(projectStaticPath, { index: ['index.html']}) + + let results = staticHandler(req.request,req.response, req.request.next) + + console.log('returning results', Object.keys(req.request)) + + + + console.log('sent already - headers?', req.response.headersSent) + console.log('sent already - date?', req.response.sendDate) + console.log('sent already - outputSize?', req.response.outputSize) + + } + + projectRunner.router.add('static-files1', '/:path*', handler) + projectRunner.router.add('static-files2', '/', handler) + + + + + await runnerRouter.addRunner({ + domain: project.data.project.domain, + runner: projectRunner + }) + } + } + } + + + + this.context.exiting = false + + return + } +} + +module.exports = VenuedHost diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js new file mode 100755 index 0000000..7ce2db9 --- /dev/null +++ b/src/venue/bin/venue.js @@ -0,0 +1,158 @@ +#!/usr/bin/env node + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree +const prompt = require('prompt') +const argon2 = require('argon2') +const OS = require('os') +const Path = require('path') + + +const Dataparty = require('../../../') + + +const commandTree = new CommandTree({ usage: 'venue [command] \nVersion: ' + Pkg.version }) + +commandTree.addCommand(require('./commands/venue-identity-gen')) +commandTree.addCommand(require('./commands/venue-identity-list')) +commandTree.addCommand(require('./commands/venue-identity-show')) + +commandTree.addCommand(require('./commands/venue-remote-add')) +commandTree.addCommand(require('./commands/venue-remote-list')) +commandTree.addCommand(require('./commands/venue-remote-show')) +commandTree.addCommand(require('./commands/venue-remote-check')) +commandTree.addCommand(require('./commands/venue-remote-repl')) + +commandTree.addCommand(require('./commands/pkg-build')) +commandTree.addCommand(require('./commands/project-build')) + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = '.venue' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + +let secureConfig = null + +async function collectPassword(info=''){ + let password = '' + + while(1){ + let passes = await prompt.get({ + properties: { + password1: { + message: 'Set'+info+' password', + hidden: true + }, + password2: { + message: 'Confim'+info+' password', + hidden: true + } + } + }) + + if(passes.password1 == passes.password2){ + + password = passes.password1 + break + } + + console.log("passwords don't match") + } + + return password +} + +async function onSetupRequired(){ + + console.log('setup-required') + + const password = await collectPassword(' keychain') + + await secureConfig.setPassword(password, { + created: Date.now() + }) + + console.log('password set') + + + await secureConfig.unlock(password) +} + +let context = { + exiting: true +} + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + let config = new Dataparty.Config.JsonFileConfig({basePath: DEFAULT_PATH}) + secureConfig = new Dataparty.Config.SecureConfig({ + config, + timeoutMs: 60*1000*5, + argon: argon2 + }) + + secureConfig.on('setup-required', onSetupRequired) + + console.log('starting') + + await config.start() + await secureConfig.start() + + + if(await secureConfig.isInitialized() && secureConfig.isLocked()){ + + const {password} = await prompt.get({ + properties: { + password: { + message: 'Enter password', + hidden: true + } + }}) + + await secureConfig.unlock(password) + } + + + await secureConfig.waitForUnlocked('startup') + + context = { + secureConfig, collectPassword, + exiting: true + } + + const output = await commandTree.run({context}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) \ No newline at end of file diff --git a/src/venue/bin/venued.js b/src/venue/bin/venued.js new file mode 100755 index 0000000..fbe37cd --- /dev/null +++ b/src/venue/bin/venued.js @@ -0,0 +1,82 @@ +#!/usr/bin/env node + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree + + +const commandTree = new CommandTree({ usage: 'venued [command] \nVersion: ' + Pkg.version }) + +commandTree.addCommand(require('./commands/venued-host')) +commandTree.addCommand(require('./commands/venued-admin-add')) +commandTree.addCommand(require('./commands/venued-admin-list')) + +/*commandTree.addCommand(require('./project/project-init')) +commandTree.addCommand(require('./project/project-show')) +commandTree.addCommand(require('./project/project-mount')) +commandTree.addCommand(require('./developer/developer-add')) +commandTree.addCommand(require('./team/team-add')) +commandTree.addCommand(require('./cloud/cloud-add')) +commandTree.addCommand(require('./cloud/cloud-list')) +commandTree.addCommand(require('./package/package-add')) +commandTree.addCommand(require('./service/service-add'))*/ + + +/* +venued host --path /opt/venue +venued get version/identity/config +venued admin add/rm/list + +venue remote add --url [] --ws [] --identity <> +venue remote rm +venue remote switch + +venue package build --output [] +venue package deploy + +venue project build +venue project deploy +*/ + +let context = { + exiting: true +} + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + const output = await commandTree.run({context}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) diff --git a/src/venue/build.js b/src/venue/build.js index 756f098..04e0416 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -1,12 +1,145 @@ +const fs = require('fs') const Path = require('path') const debug = require('debug')('build') const Dataparty = require('../index.js') +const dataparty_crypto = require('@dataparty/crypto') + const Pkg = require('../../package.json') const VenueService = require('./venue-service') +/** + * build/ + * - service.venue.json + * - schema.venue.json + * - client.venue.json + * - staticFiles.venue.tgz + * - package.venue.json + * { author, venue, files[], signatures{} } + */ + + +/* + +venue: { + url, + publicKey, + +} + +.venue-admin/ + - config/config.json + - db/ + - packages/ + - AUTHOR/PACKAGE-NAME + - package.venue.json + - service.venue.json + - static-files.venue.tgz + - parties/ + - PARTY_PUBLIC/ + - config/config.json (optional) + - db/ (optional) + - static-files.venue.tgz + - projects/ + - AUTHOR/PROJECT_PUBLIC + - project.venue.json + - static-files.venue.tgz + - party/ + - PARTY_NAME/ + - static-files.venue.tgz + + +~/code/my_venue_project/ + - package.json + - build.js + - public/ + - party/NAME + - default-config.json + - public/ + - package/NAME + - project/NAME + - venue.json + { + projects: { + NAME: { + name: string + domain, (optional) + venue, (optional) + parties: [NAME], + routes: { + PREFIX: [ { PACKAGE(owner,name,version), party } ] + } + } + }, + packages: { + NAME: { + name: String + version: String, (optional) + service: localPathToService.js OR built service.json + } + } + } + +*/ + +async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, existingBuildPath, staticFilePaths}){ + + /* + + 0. create service.venue.json + 1. create schema.venue.json + 2. create client.venue.json + 3. create staticFiles.venue.tgz + 4. create package.venue.json + + 6. upload to venue + - create-package + * service.venue.json + * package.venue.json + + 7. upload files to venue + - create-files + * staticFiles.venue.tgz + * schema.venue.json + * client.venue.json + + 8. + + */ + +} + + +/** + * + * 1. generate admin/developer key + * 2. add venue identity (by url) + * 3. venue package build - build a package at some path + * 4. venue package push - c + */ + +async function pushService(devId, build, staticTar){ + + let client = new Dataparty.MatchMakerClient( + devId, + null, + 'https://api.dataparty.xyz/venue', + 'wss://api.dataparty.xyz/ws' + ) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) +} async function main(){ const service = new VenueService({ @@ -14,13 +147,36 @@ async function main(){ version: Pkg.version }) + const path = Path.join(process.env.HOME, '.venue-admin') + + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + let party = new Dataparty.TingoParty({ + path: path+'/db', + config, + noCache: false + }) + + await party.start() + + console.log( 'identity - ', party.identity.key.hash ) + const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(Path.join(__dirname,'./dataparty'), true) + const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, party.privateIdentity) debug('compiled') + + const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') + + debug('is staticTar a buffer? ', staticTar instanceof Buffer); // true + await pushService( party.privateIdentity, build, staticTar ) + + } main().catch(err=>{ console.error('CRASH') console.error(err) -}) \ No newline at end of file +}) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js new file mode 100644 index 0000000..18d05d9 --- /dev/null +++ b/src/venue/endpoints/create-package.js @@ -0,0 +1,353 @@ +const fs = require('fs') +const Joi = require('joi') +const Path = require('path') +const Hoek = require('@hapi/hoek') +const {Message, Routines, Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-package') +const zlib = require('zlib') + +const IEndpoint = require('../../service/iendpoint') + +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + +module.exports = class CreatePkgEndpoint extends IEndpoint { + + static get Name(){ + return 'create-package' + } + + + static get Description(){ + return 'Create venue package' + } + + /* + { + venue_package: { + package:{ + owner: String, + info: { + name, version, githash, branch + }, + files: [ //package, service, static.tgz, + {hash: String, name: String, size: Number, signature} + ], + statics: { + PREFIX: [localPathGlob] + } + }, + trust: { + owner: signature + } + + } + } + + { + venue_project: { + + project:{ + + owner: String, + domain: String, + venue: String, + + parties: { + NAME: { + keys: {public, private: Secret(private)}, + defaultConfig: Object, + files: [ //static.tgz, + {hash: String, name: String, signature} + ], + statics: {prefix, [localPath]} + } + }, + + + + + routes: { + PREFIX: [{ + party: String, + package: { + owner: String, + info: {name, version, branch, githash}, + settings: { + sendFullErrors, + useNative + } + } + }] + } + }, + trust: { + owner: signature + } + } + } + + */ + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + ephemeral_session: true, + validate: Joi.object().keys({ + settings: Joi.object().keys({ + //enabled: Joi.boolean().default(true).required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), + sendFullErrors: Joi.boolean().default(false), + useNative: Joi.boolean().default(false), + defaultConfig: Joi.object().keys(null) + }), + build: Joi.object().keys({ + package: Joi.object().keys({ + owner: Joi.string(), + name: Joi.string().required(), + version: Joi.string().required(), + githash: Joi.string().required(), + branch: Joi.string().required() + }).required(), + schemas: Joi.object().keys(null), + documents: Joi.object().keys(null), + endpoints: Joi.object().keys(null), + middleware: Joi.object().keys(null), + middleware_order: Joi.object().keys(null), + tasks: Joi.object().keys(null), + topics: Joi.object().keys(null), + auth: Joi.object().keys(null), + files: Joi.object().keys(null), + signatures: Joi.object().keys(null).required(), + compileSettings: Joi.object().keys(null) + }).required(), + //staticTar: Joi.binary() + staticTar: Joi.any().custom(typedArraySchema) + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + let {signatures, ...buildWithoutSig} = ctx.input.build + + const pkgOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.build.package.owner ) + .exec() + )[0] + + if(!pkgOwnerIdentityDoc){ + throw new Error('package owner not authorized') + } + + debug('found pkg owner', pkgOwnerIdentityDoc.hash) + + const pkgOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: pkgOwnerIdentityDoc.data.type, + hash: pkgOwnerIdentityDoc.data.hash, + public: pkgOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + //debug('build', buildWithoutSig) + + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.build.package.owner]) + + let signedBuildMsg = new Message({ + msg: buildWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') + + await signedBuildMsg.assertVerified(pkgOwnerIdentity, true) + + debug('verified package signature') + + const safeFileName = ctx.input.build.package.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' + + if(ctx.input.staticTar){ + + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const buildFiles = ctx.input.build.files[tarFileName] + + if(buildFiles && buildFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match package definition") + } + + debug('verified staticTar') + } + + debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) + + const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(/*ctx.input.build*/buildWithoutSig) + + const buildHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + buildBSON + ) + ) + + const safeBuildHash = buildHash.replace(/\//g, "-") + + debug('\t'+'hash', buildHash) + + const buildWorkspace = Path.join('packages', safeFileName, ctx.input.build.package.version, safeBuildHash) + + const config = ctx.party.config + + const workspacePath = await config.touchDir(buildWorkspace) + + + debug('\t'+'workspace - local', buildWorkspace) + debug('\t'+'workspace - global', workspacePath) + + + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.build)) + + const build = ctx.input.build + const serviceId = build.package.name + '@' + build.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_pkg') + .where('package.name').equals(build.package.name) + .where('package.version').equals(build.package.version) + .where('hash').equals(buildHash) + .exec())[0] + + + if(!srvDoc){ + debug('creating service') + + const {owner, ...pkgWithoutOwner} = build.package + + srvDoc = await ctx.party.createDocument('venue_pkg', { + owner: build.package.owner, + 'created': Date.now(), + venue: ctx.party.identity.key.hash, + hash: buildHash, + workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), + settings: ctx.input.settings, + package: pkgWithoutOwner, + compressedBuild: Routines.Utils.base64.encode(compressedBrotliBuild) + }) + + debug('service created') + } else { + debug('need to update service?') + } + + if(ctx.input.staticTar){ + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + } + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) + + // verify build signature + + //ctx.input. + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + + const compiledSrv = JSON.parse(ctx.input.service) + const serviceId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!srvDoc){ + debug('creating service') + srvDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(srvDoc.data) + await srvDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(srvDoc.data) + + debug('saving doc') + + + await srvDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + }*/ + + return {done: true, package: srvDoc.data} + + //return {srv:srvDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js new file mode 100644 index 0000000..83c1f2a --- /dev/null +++ b/src/venue/endpoints/create-project.js @@ -0,0 +1,404 @@ +const fs = require('fs') +const Joi = require('joi') +const Path = require('path') +const Hoek = require('@hapi/hoek') +const {Message, Routines, Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-project') + +const process = require('process') +const tar = require('tar') +const zlib = require('zlib') + +const IEndpoint = require('../../service/iendpoint') + +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + +module.exports = class CreateProjectEndpoint extends IEndpoint { + + static get Name(){ + return 'create-project' + } + + + static get Description(){ + return 'Create venue project' + } + + + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + ephemeral_session: true, + validate: Joi.object().keys({ + project: Joi.object().keys({ + owner: Joi.string().required(), + //created: Joi.number(), + + name: Joi.string().required(), + version: Joi.string().required(), + venue: Joi.string().required(), + domain: Joi.string(), + + i2p: Joi.object().keys({ + address: Joi.string(), + public: Joi.string(), + securePrivate: Joi.string() + }), + party: Joi.array().items(Joi.object().keys({ + name: Joi.string(), + type: Joi.string().required(), + tingo: { path: Joi.string() }, + loki: { path: Joi.string() }, + peer: { + venue: Joi.string(), + remoteIdentity: Joi.string() + }, + key: { + hash: Joi.string(), + securePrivate: Joi.string() + }, + settings: { noCache: Joi.string() }, + defaultConfig: Joi.string() + })), + routes: Joi.array().items(Joi.object().keys({ + prefix: Joi.string(), + party: Joi.string(), + package: Joi.object().keys({ + owner: Joi.string(), + name: Joi.string().required(), + version: Joi.string(), + branch: Joi.string(), + hash: Joi.string() + }), + settings: { + sendFullErrors: Joi.boolean().required(), + useNative: Joi.boolean().required() + } + })), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + tar: Joi.string(), + hash: Joi.string().required(), + size: Joi.number().required(), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + hash: Joi.string().required(), + size: Joi.number().required() + })) + })), + signatures: Joi.object().pattern(Joi.string(), Joi.string()).required() + }).required(), + staticTar: Joi.any().custom(typedArraySchema) + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + if(ctx.party.identity.key.hash != ctx.input.project.venue){ + throw new Error('project venue does not match this host') + } + + let {signatures, ...projectWithoutSig} = ctx.input.project + + const projectOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.project.owner ) + .exec() + )[0] + + if(!projectOwnerIdentityDoc){ + throw new Error('project owner not authorized') + } + + debug('found project owner', projectOwnerIdentityDoc.hash) + + const projectOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: projectOwnerIdentityDoc.data.type, + hash: projectOwnerIdentityDoc.data.hash, + public: projectOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.project.owner]) + + let signedProjectMsg = new Message({ + msg: projectWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') + + await signedProjectMsg.assertVerified(projectOwnerIdentity, true) + + debug('verified package signature') + + const safeFileName = ctx.input.project.name.replace('/', '-') + const tarFileName = safeFileName+'.project.files.venue.tgz' + + const projectFiles = ctx.input.project.files[tarFileName] + + if(projectFiles && !ctx.input.staticTar){ + throw new Error('project definition lists a static tar but none was uploaded') + } + + if(ctx.input.staticTar){ + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + if(projectFiles && projectFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match project definition") + } + + debug('verified staticTar') + } + + // check route packages are valid + let packages = {} + let tarList = [] + + for(let route of ctx.input.project.routes){ + console.log(route) + let pkgDoc = (await ctx.party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + let pkgTarPath = pkgDoc.data.tarpath + + if(pkgTarPath && pkgTarPath.length > 0){ + tarList.push(pkgTarPath) + } + + packages[route.prefix] = pkgDoc.data + } + + + debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) + + const projectBSON = Routines.BSON.serializeBSONWithoutOptimiser(projectWithoutSig) + + const projectHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + projectBSON + ) + ) + + const safeProjectHash = projectHash.replace(/\//g, "-") + + debug('\t'+'hash', projectHash) + + const projectWorkspace = Path.join('projects/',safeFileName, ctx.input.project.version, safeProjectHash) + + const config = ctx.party.config + + const workspacePath = await config.touchDir(projectWorkspace) + + + debug('\t'+'workspace - local', projectWorkspace) + debug('\t'+'workspace - global', workspacePath) + + + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.project)) + + const project = ctx.input.project + const projectId = project.name + '@' + project.version + debug('addProject', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_project') + .where('project.name').equals(project.name) + .where('project.version').equals(project.version) + .where('hash').equals(projectHash) + .exec())[0] + + + if(!projectDoc){ + debug('creating project') + + const {owner, ...pkgWithoutOwner} = project + + projectDoc = await ctx.party.createDocument('venue_project', { + owner: project.owner, + created: Date.now(), + changed: Date.now(), + hash: projectHash, + workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), + project: ctx.input.project, + }) + + debug('project created') + } else { + debug('need to update project?') + } + + debug('extracting other tars', tarList) + + for(let tarPath of tarList){ + debug('extracting', tarPath) + + await tar.extract({ + cwd: workspacePath, + file: tarPath, + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, /*tarFileList*/ ) + } + + if(ctx.input.staticTar){ + debug('saving tar file', tarFileName) + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + debug('extracting contents') + + /*await tar.t({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + onReadEntry: entry => { console.log('\t\t', entry) } + })*/ + + //const tarFileInfo = projectDoc.data.project.files[ tarFileName ] + + //if(projectFiles){ + const tarFileList = Object.keys(projectFiles.files) + + debug('file list - ', tarFileList) + + await tar.extract({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, tarFileList ) + + //} + } + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.project.venue.json'), + JSON.stringify(ctx.input.project, null, 2) + ) + + await ctx.party.config.write('projects.'+ctx.input.project.name, projectHash) + + // verify build signature + + //ctx.input. + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + + const compiledSrv = JSON.parse(ctx.input.service) + const projectId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!projectDoc){ + debug('creating service') + projectDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(projectDoc.data) + await projectDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(projectDoc.data) + + debug('saving doc') + + + await projectDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + }*/ + + console.log('restarting in 3 seconds...') + setTimeout(()=>{ + process.exit(), + 3000 + }) + + return {done: true, project: projectDoc.data} + + //return {srv:projectDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/endpoints/create-service.js b/src/venue/endpoints/file-create.js similarity index 89% rename from src/venue/endpoints/create-service.js rename to src/venue/endpoints/file-create.js index ac3e324..d2e20d6 100644 --- a/src/venue/endpoints/create-service.js +++ b/src/venue/endpoints/file-create.js @@ -1,19 +1,19 @@ const Joi = require('joi') const Hoek = require('@hapi/hoek') const {Message, Routines} = require('@dataparty/crypto') -const debug = require('debug')('dataparty.endpoint.create-service') +const debug = require('debug')('dataparty.endpoint.file-create') const IEndpoint = require('../../service/iendpoint') -module.exports = class CreateSrvEndpoint extends IEndpoint { +module.exports = class FileCreateEndpoint extends IEndpoint { static get Name(){ - return 'create-service' + return 'file-create' } static get Description(){ - return 'Create venue service' + return 'Create venue package file' } static get MiddlewareConfig(){ @@ -21,13 +21,13 @@ module.exports = class CreateSrvEndpoint extends IEndpoint { pre: { decrypt: true, validate: Joi.object().keys({ - settings: Joi.object().keys({ + /*settings: Joi.object().keys({ enabled: Joi.boolean().default(true).required(), - domain: Joi.string().required(), - prefix: Joi.string().default('').required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), sendFullErrors: Joi.boolean().default(false).required(), useNative: Joi.boolean().default(false).required() - }).required(), + }).required(),*/ service: Joi.object().keys({ package: Joi.object().keys({ name: Joi.string().required(), diff --git a/src/venue/endpoints/file-download.js b/src/venue/endpoints/file-download.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/endpoints/file-write.js b/src/venue/endpoints/file-write.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/example-project.js b/src/venue/example-project.js new file mode 100644 index 0000000..315dd1f --- /dev/null +++ b/src/venue/example-project.js @@ -0,0 +1,25 @@ +module.exports = { + owner: 'null', + name: 'example-project', + version: '1.0', + + venue: 'dataparty-venue', + domain: 'api.dataparty.xyz', + + routes: [{ + prefix: '/api', + party:'SYSTEM', + package: { + name: '@dataparty/api' + }, + settings: { + sendFullErrors: false, + useNative: false + } + }], + files: [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ] + } \ No newline at end of file diff --git a/src/venue/middleware/pre/decrypt-nacl.js b/src/venue/middleware/pre/decrypt-nacl.js index 069812a..857f2e9 100644 --- a/src/venue/middleware/pre/decrypt-nacl.js +++ b/src/venue/middleware/pre/decrypt-nacl.js @@ -32,7 +32,7 @@ module.exports = class DecryptNaCl extends IMiddleware { if (!Config){ return } if(!context.input || !context.input.enc){ - throw new Error('insecure message') + throw new Error('insecure message - here') } context.debug('input', context.input, typeof context.input) diff --git a/src/venue/public/p2p-test.html b/src/venue/public/p2p-test.html index 6c57aa8..a45f222 100644 --- a/src/venue/public/p2p-test.html +++ b/src/venue/public/p2p-test.html @@ -256,7 +256,12 @@

Offers recieved

await hostLocal.start() - matchMaker = new Dataparty.MatchMakerClient(hostLocal.privateIdentity, null) + matchMaker = new Dataparty.MatchMakerClient( + hostLocal.privateIdentity, + null, + 'https://api.dataparty.xyz/api', + 'wss://api.dataparty.xyz/ws' + ) await matchMaker.start() diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js new file mode 100644 index 0000000..8ce3914 --- /dev/null +++ b/src/venue/schema/venue_package.js @@ -0,0 +1,55 @@ +'use strict' + +const debug = require('debug')('venue.venue_pkg') + +const ISchema = require('../../bouncer/ischema') + +//const Utils = ISchema.Utils + + +class VenuePkg extends ISchema { + + static get Type () { return 'venue_pkg' } + + static get Schema(){ + return { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + venue: {type: String}, + workspace: {type: String, required: true}, + tarpath: {type: String}, + hash: {type: String, required: true, index: true}, + settings: { + //enabled: {type: Boolean, required: true}, + staticPrefix: String, + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + defaultConfig: {type: Object} + }, + package: { + name: {type: String, required: true, index: true}, + version: {type: String, required: true, index: true}, + githash: {type: String, required: true}, + branch: {type: String, required: true}, + }, + compressedBuild: {type: String, required: true} //! brotli compressed + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenuePkg \ No newline at end of file diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js new file mode 100644 index 0000000..2d65721 --- /dev/null +++ b/src/venue/schema/venue_project.js @@ -0,0 +1,100 @@ +'use strict' + +const debug = require('debug')('venue.venue_project') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenueProject extends ISchema { + + static get Type () { return 'venue_project' } + + static get Schema(){ + return { + owner: {type: String, required: true}, + created: {type: Number, required: true}, + changed: {type: Number, required: true}, + workspace: {type: String, required: true}, + tarpath: {type: String}, + hash: {type: String, required: true, index: true}, + + enabled: {type: Boolean}, + + + project: { + owner: {type: String, required: true, index: true}, //public_key.key.hash + //created: {type: Number, required: true}, + + name: {type: String, required: true, index: true}, + version: {type: String, required: true, index: true}, + venue: {type: String}, + domain: {type: String, index: true/*, unique: true*/}, + + i2p: { + address: String, + public: String, + securePrivate: String + }, + + party: [{ + name: String, + type: {type: String, enum: ['tingo', 'loki', 'peer', 'mongo']}, + tingo: { + path: String + }, + loki: { + path: String + }, + peer: { + venue: String, + remoteIdentity: String + }, + key: { + hash: String, + securePrivate: String + }, + settings: { + noCache: Boolean, + }, + defaultConfig: String + }], + routes: [{ + prefix: String, + party: String, + package: { + owner: String, + name: {type: String, required: true}, + version: String, + branch: String, + hash: String, + }, + settings: { + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + } + }], + files: Object, + signatures: {type: Object, required: true} + } + + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenueProject \ No newline at end of file diff --git a/src/venue/schema/venue_service.js b/src/venue/schema/venue_service.js index f5f8d08..181a556 100644 --- a/src/venue/schema/venue_service.js +++ b/src/venue/schema/venue_service.js @@ -2,7 +2,6 @@ const debug = require('debug')('venue.venue_srv') - const ISchema = require('../../bouncer/ischema') const Utils = ISchema.Utils @@ -31,12 +30,6 @@ class VenueSrv extends ISchema { version: {type: String, required: true}, githash: {type: String, required: true}, branch: {type: String, required: true} - }, - compressedBuild: {type: String, required: true}, //! zlib compressed - signature: { - timestamp: {type: Number, required: true}, - type: {type: String, required: true, maxlength: 10}, - value: {type: String, required: true} } } } diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index fef4471..6e72731 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -5,8 +5,8 @@ const Dataparty = require('../index') const VenueService = require('./venue-service') -const VenueServiceSchema = require('./dataparty/@dataparty-venue.dataparty-schema.json') -const VenueSrv = require('./dataparty/@dataparty-venue.dataparty-service.json') +const VenueServiceSchema = require('./dataparty/@dataparty-venue.schema.venue.json') +const VenueSrv = require('./dataparty/@dataparty-venue.service.venue.json') async function loadService(runnerRouter, settings, serviceFilePath){ @@ -55,7 +55,8 @@ async function main(){ const CloudFlareIpFilter = { options: { - mode: 'allow' + mode: 'allow', + //trustProxy: true }, ips: [ '173.245.48.0/20', @@ -79,14 +80,15 @@ async function main(){ '2405:b500::/32', '2405:8100::/32', '2a06:98c0::/29', - '2c0f:f248::/32' + '2c0f:f248::/32', + '10.115.68.55/32' ] } let party = new Dataparty.TingoParty({ path: path+'/db', model: VenueServiceSchema, - config: new Dataparty.Config.JsonFileConfig({basePath: path+'/config'}), + config: new Dataparty.Config.JsonFileConfig({basePath: path}), noCache: false }) @@ -101,7 +103,7 @@ async function main(){ party, service, sendFullErrors: true, useNative: false, - prefix: 'api/' + prefix: 'venue/' }) let runnerRouter = new Dataparty.RunnerRouter(runner) @@ -114,7 +116,7 @@ async function main(){ trust_proxy: true, wsEnabled: true, ssl_key, ssl_cert, - listenUri: 'https://0.0.0.0:443', + listenUri: 'https://0.0.0.0:3000', staticPath: Path.join(__dirname,'public'), staticPrefix: '/venue/', ipFilter: CloudFlareIpFilter @@ -126,7 +128,7 @@ async function main(){ debug('started') console.log('partying') - +/* await loadService(runnerRouter, { enabled: true, domain: 'postquantum.one', @@ -135,7 +137,7 @@ async function main(){ sendFullErrors: true, useNative: false - }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json') + }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json')*/ } diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index f5c7ad3..e98511e 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -11,19 +11,38 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) - builder.addSchema(Path.join(__dirname, './schema/venue_service.js')) + + builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) + builder.addSchema(Path.join(__dirname, './schema/venue_project.js')) + + builder.addSchema(DatapartySrv.schema_paths.public_key) + builder.addSchema(DatapartySrv.schema_paths.session_key) builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) + builder.addMiddleware(DatapartySrv.middleware_paths.pre.ephemeral_session) + builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) builder.addEndpoint(DatapartySrv.endpoint_paths.identity) builder.addEndpoint(DatapartySrv.endpoint_paths.version) + builder.addEndpoint(DatapartySrv.endpoint_paths.key_announce) + + builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) + + + builder.addTask(DatapartySrv.task_paths.cleanup_ephemeral_sessions) - builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) builder.addAuth(Path.join(__dirname, './auth.js')) + + builder.addFiles(__dirname, [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ], { nodir: true, follow: true }) } }