diff --git a/README.md b/README.md index a01db21..4287c5a 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,60 @@ example `Application.version` in Unity, or the assembly's informational version elsewhere. Pass `key`, `enabled` and `log` by name, as above, so a key can never land in the version's place. +## Every event carries a random installation ID + +Since 0.3.0, every event can also carry the tag `install`: a random ID for +the installation, so the trace server can count **distinct installations** +("active installs in the last 30 days") rather than raw events. This is the +same idea as bStats' `serverUuid`, and it is said out loud here because it is +the one thing the client sends that is the same from one event to the next. + +**What it is.** A `Guid.NewGuid()`. It is not derived from anything — not a +hostname, an IP address, a MAC address, a player, an account or a path. It +identifies no person and no address; all it can say is "these events came +from the same installation". (The trace server still sees the IP address of +every HTTP request, as every web server does.) + +**Where it lives.** Wherever the program decides — there is no hidden +default location, and without one of the options below no ID is made up and +nothing is written. The simplest way is a file the program chooses: + +```csharp +var trace = new TraceClient(url, "my-game", "1.4.0", + key: settings.UsageReportingKey, + enabled: settings.UsageReportingEnabled, + installIdFile: Path.Combine(settings.DataDirectory, "trace-install-id")); +``` + +The client reads the first line of that file that is 1–255 of +`[A-Za-z0-9_.-]`; when the file is missing or holds no such line, it writes a +new random ID there (creating parent directories). If the file cannot be read +or written, a fresh ID is used in memory for that run only — the constructor +still never throws for it. `TraceClient.InstallIdFromFile(path)` does the same +on its own, but note that calling it directly reads and writes the file +whatever the opt-outs say; passing `installIdFile:` lets the client do it only +when reporting is on. + +A program that already keeps settings can store the ID there instead and pass +it explicitly; `installId:` wins over `installIdFile:`: + +```csharp +var trace = new TraceClient(url, "my-game", "1.4.0", key: key, + installId: settings.UsageReportingInstallId); // null or blank: none sent +``` + +An explicit ID is trimmed; one over 255 characters throws `ArgumentException`, +like an overlong version. An event that passes its own `install` tag keeps +it, and `install` is never added past the server's 32-tag limit. +`trace.InstallId` returns the ID in use (`null` when disabled or when there is +none), so a program can print it. + +**Resetting it.** Delete the file (or the setting); the next start makes a new +one. Or put your own value in it. + +**Opting out.** Every [opt-out](#turning-it-off) also stops the ID: a disabled +client never generates one, never writes one, and sends nothing. + ## What `Report` promises | Property | Meaning | @@ -104,14 +158,15 @@ distribution. ## The wire format `POST {baseUrl}/api/metrics` with `Authorization: Bearer `, -`User-Agent: trace-client/0.2.0 ()` and a body of +`User-Agent: trace-client/0.3.0 ()` and a body of ```json -{"application":"my-game","name":"level-complete","value":42,"tags":{"version":"1.4.0"}} +{"application":"my-game","name":"level-complete","value":42,"tags":{"version":"1.4.0","install":"0f8b6c1e-3a52-4c8e-9a0d-6e2f1b7c4d90"}} ``` `value` is omitted when not given, and `tags` always holds at least -`version`. `value` is written with the invariant culture, so a German locale +`version`, plus `install` when the program gave the client an +[installation ID](#every-event-carries-a-random-installation-id). `value` is written with the invariant culture, so a German locale still sends `2.5`. The server assigns the timestamp. A `201` is success; anything else is logged and dropped. diff --git a/src/TraceClient/TraceClient.cs b/src/TraceClient/TraceClient.cs index dd3e2f1..06241b6 100644 --- a/src/TraceClient/TraceClient.cs +++ b/src/TraceClient/TraceClient.cs @@ -1,5 +1,5 @@ /* - * trace-client 0.2.0 -- https://github.com/Stephenson-Software/trace-client-csharp + * trace-client 0.3.0 -- https://github.com/Stephenson-Software/trace-client-csharp * * One call to report that a program was used. Copy this file into a project as * is, or reference the project; either way there is nothing else to add. @@ -12,8 +12,10 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using System.Threading; namespace StephensonSoftware.Trace @@ -45,6 +47,12 @@ namespace StephensonSoftware.Trace /// version -- the third constructor argument, required, so a /// command event can be tied to a release as well as a /// startup one. An event's own version tag wins over it. + /// Every event also carries a random per-installation ID as the tag + /// install when the program supplies one -- installId, or a file + /// it chooses via installIdFile (see ) -- + /// so the trace server can count distinct installations rather than raw + /// events. It is resolved only after every opt-out: a disabled client never + /// makes one up or writes one. An event's own install tag wins over it. /// /// var trace = new TraceClient("https://trace.example.org", "my-game", ProgramVersion, /// key: settings.UsageReportingKey, @@ -57,7 +65,7 @@ namespace StephensonSoftware.Trace public sealed class TraceClient : IDisposable { /// The client version, as sent in the User-Agent header. - public const string Version = "0.2.0"; + public const string Version = "0.3.0"; /// How many reports may wait to be sent before new ones are dropped. public const int QueueCapacity = 256; @@ -86,6 +94,13 @@ public sealed class TraceClient : IDisposable /// Reason given when no key was supplied. public const string ReasonNoKey = "no key"; + /// The tag every event carries the installation's ID as. + public const string InstallTag = "install"; + + // What a stored installation ID may look like: the same characters the + // Java client accepts in server-id:, at most MaxLength of them. + private static readonly Regex InstallIdLine = new Regex("^[A-Za-z0-9_.-]{1,255}$"); + // Where environment variables come from. A seam rather than // System.Environment directly, so tests can point it at a dictionary; // nothing else should touch it. @@ -118,8 +133,17 @@ public sealed class TraceClient : IDisposable /// The program's write key. Without one the client is a no-op. /// The program's own opt-out. false yields a client that reports nothing. /// Where dropped reports are mentioned. Optional; treat as debug-level. + /// The installation's ID, sent as the tag install on every + /// event. It should be random -- e.g. a the program stores in its + /// own settings -- and never derived from a person, account or address. Trimmed; null + /// or blank means none. Longer than characters throws + /// . Wins over . + /// A file, chosen by the program, that holds the installation's + /// ID; read or created with , but only when the client is + /// enabled, so a disabled client never writes it. Optional; there is no default location. public TraceClient(string baseUrl, string application, string version, string key = null, - bool enabled = true, Action log = null) + bool enabled = true, Action log = null, + string installId = null, string installIdFile = null) { if (string.IsNullOrWhiteSpace(baseUrl)) { @@ -137,6 +161,11 @@ public TraceClient(string baseUrl, string application, string version, string ke { throw new ArgumentException("version is longer than " + MaxLength + " characters", "version"); } + string explicitInstallId = installId == null ? null : installId.Trim(); + if (explicitInstallId != null && explicitInstallId.Length > MaxLength) + { + throw new ArgumentException("installId is longer than " + MaxLength + " characters", "installId"); + } try { _endpoint = new Uri(baseUrl.Trim().TrimEnd('/') + "/api/metrics"); @@ -169,6 +198,16 @@ public TraceClient(string baseUrl, string application, string version, string ke { return; } + // After the opt-outs, never before: a disabled client neither makes + // up an ID nor writes one to disk. + if (!string.IsNullOrEmpty(explicitInstallId)) + { + InstallId = explicitInstallId; + } + else if (!string.IsNullOrWhiteSpace(installIdFile)) + { + InstallId = ReadOrCreateInstallId(installIdFile, Log); + } _queue = new BlockingCollection(new ConcurrentQueue(), QueueCapacity); _stop = new CancellationTokenSource(); _http = new HttpClient { Timeout = Timeout }; @@ -190,6 +229,92 @@ public static TraceClient Disabled() /// public string DisabledReason { get; private set; } + /// + /// The random per-installation ID every event carries as the tag install, + /// or null when the client is disabled or was given none (no + /// installId and no installIdFile). Decided once, in the constructor. + /// + public string InstallId { get; private set; } + + /// + /// The installation ID stored in , creating it if needed: + /// the first line that is 1-255 of [A-Za-z0-9_.-] is the ID. When the file + /// is missing or holds no such line, a new random is written to + /// it (parent directories created) and returned. When the file cannot be read or + /// written -- or is blank -- a new random ID is returned + /// for this process only. Never throws. There is no default location: the program + /// chooses where its ID lives, and deleting the file resets it. + /// + /// + /// Calling this directly reads and writes the file whatever the opt-outs say. + /// Pass the path as the constructor's installIdFile instead to keep the + /// promise that a disabled client writes nothing. + /// + public static string InstallIdFromFile(string path) + { + return ReadOrCreateInstallId(path, null); + } + + private static string ReadOrCreateInstallId(string path, Action log) + { + string fresh = Guid.NewGuid().ToString("D"); + if (string.IsNullOrWhiteSpace(path)) + { + return fresh; + } + try + { + foreach (string line in File.ReadAllLines(path)) + { + string candidate = line.Trim(); + if (InstallIdLine.IsMatch(candidate)) + { + return candidate; + } + } + } + catch (Exception failure) when (failure is FileNotFoundException || failure is DirectoryNotFoundException) + { + // nothing stored yet; write one below + } + catch (Exception failure) + { + // A file that is there but cannot be read is not overwritten. + Note(log, "using an in-memory install ID for this process: could not read " + path + ": " + failure.Message); + return fresh; + } + try + { + string directory = Path.GetDirectoryName(Path.GetFullPath(path)); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + File.WriteAllText(path, fresh + "\n"); + } + catch (Exception failure) + { + Note(log, "using an in-memory install ID for this process: could not write " + path + ": " + failure.Message); + } + return fresh; + } + + private static void Note(Action log, string message) + { + if (log == null) + { + return; + } + try + { + log("[trace] " + message); + } + catch (Exception) + { + // a throwing logger must not break the promise either + } + } + /// Whether will actually send anything. false after too. public bool IsEnabled { @@ -252,7 +377,7 @@ public void Report(string name, double? value = null, IEnumerable @@ -406,6 +520,37 @@ internal static List> WithVersion( return merged; } + /// + /// The tags plus install, unless they already carry one, there is no + /// ID, or adding it would pass . Modifies and returns + /// , which is already a copy from . + /// + internal static List> WithInstall( + List> tags, string installId) + { + if (installId == null) + { + return tags; + } + int sent = 0; // counted the way Json counts: blank keys and null values are skipped + foreach (KeyValuePair tag in tags) + { + if (tag.Key == InstallTag) + { + return tags; + } + if (!string.IsNullOrWhiteSpace(tag.Key) && tag.Value != null) + { + sent++; + } + } + if (sent < MaxTags) + { + tags.Add(new KeyValuePair(InstallTag, installId)); + } + return tags; + } + // JSON is written by hand so this file has no dependencies. The shape is // fixed and small -- three scalars and a flat string map. Returns null // and sets body when the report is sendable, otherwise the reason it is not. diff --git a/src/TraceClient/TraceClient.csproj b/src/TraceClient/TraceClient.csproj index fedffa5..ed9d303 100644 --- a/src/TraceClient/TraceClient.csproj +++ b/src/TraceClient/TraceClient.csproj @@ -10,7 +10,7 @@ 7.3 StephensonSoftware.Trace StephensonSoftware.Trace - 0.2.0 + 0.3.0 true true diff --git a/tests/TraceClient.Tests/InstallIdTest.cs b/tests/TraceClient.Tests/InstallIdTest.cs new file mode 100644 index 0000000..43ea8cc --- /dev/null +++ b/tests/TraceClient.Tests/InstallIdTest.cs @@ -0,0 +1,271 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using Xunit; + +namespace StephensonSoftware.Trace.Tests +{ + /// + /// The per-installation ID sent as the tag install: the same cases + /// the Java client's suite covers, plus the file helper. + /// + public sealed class InstallIdTest : IDisposable + { + private readonly Dictionary _environment = new Dictionary(); + private readonly Func _realEnvironment; + private readonly StubServer _server = new StubServer(); + private readonly string _directory = Path.Combine(Path.GetTempPath(), "trace-install-" + Guid.NewGuid().ToString("N")); + + public InstallIdTest() + { + _realEnvironment = TraceClient.EnvironmentSource; + TraceClient.EnvironmentSource = name => _environment.TryGetValue(name, out string v) ? v : null; + } + + public void Dispose() + { + TraceClient.EnvironmentSource = _realEnvironment; + _server.Dispose(); + try + { + Directory.Delete(_directory, true); + } + catch (Exception) + { + // nothing was made + } + } + + private static bool IsGuid(string value) + { + return Guid.TryParseExact(value, "D", out Guid _); + } + + [Fact] + public void InstallIdFromFile_WritesOneIdOnceAndReusesIt() + { + string path = Path.Combine(_directory, "nested", "dirs", "install-id"); + + string first = TraceClient.InstallIdFromFile(path); + string second = TraceClient.InstallIdFromFile(path); + + Assert.True(IsGuid(first), first); + Assert.Equal(first, second); + Assert.Equal(first + "\n", File.ReadAllText(path)); // parent directories were created + } + + [Fact] + public void InstallIdFromFile_ReadsTheFirstValidLineAndReplacesAFileWithoutOne() + { + Directory.CreateDirectory(_directory); + string kept = Path.Combine(_directory, "kept"); + File.WriteAllText(kept, "\n not valid! \n my-own_id.1 \nsecond\n"); + string junk = Path.Combine(_directory, "junk"); + File.WriteAllText(junk, "has spaces\n" + new string('a', TraceClient.MaxLength + 1) + "\n"); + + Assert.Equal("my-own_id.1", TraceClient.InstallIdFromFile(kept)); + string fresh = TraceClient.InstallIdFromFile(junk); + Assert.True(IsGuid(fresh), fresh); + Assert.Equal(fresh, TraceClient.InstallIdFromFile(junk)); + } + + [Fact] + public void InstallIdFromFile_FallsBackToAnInMemoryIdWithoutThrowing() + { + // A path under a regular file can be neither read nor created, even as root. + Directory.CreateDirectory(_directory); + string blocker = Path.Combine(_directory, "a-file"); + File.WriteAllText(blocker, "x"); + string unwritable = Path.Combine(blocker, "install-id"); + + string first = TraceClient.InstallIdFromFile(unwritable); + string second = TraceClient.InstallIdFromFile(unwritable); + + Assert.True(IsGuid(first), first); + Assert.NotEqual(first, second); // nothing persisted + Assert.Equal("x", File.ReadAllText(blocker)); + Assert.True(IsGuid(TraceClient.InstallIdFromFile(null))); + Assert.True(IsGuid(TraceClient.InstallIdFromFile(" "))); + } + + [Fact] + public void InstallIdFromFile_DoesNotOverwriteAPathItCannotRead() + { + // A directory where the file should be: unreadable as a file, and left alone. + string directoryInTheWay = Path.Combine(_directory, "install-id"); + Directory.CreateDirectory(directoryInTheWay); + + string id = TraceClient.InstallIdFromFile(directoryInTheWay); + + Assert.True(IsGuid(id), id); + Assert.True(Directory.Exists(directoryInTheWay)); + } + + [Fact] + public void Client_PersistsItsIdInTheGivenFileAndSendsItOnEveryEvent() + { + string path = Path.Combine(_directory, "install-id"); + var client = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installIdFile: path); + string id = client.InstallId; + + client.Report("startup"); + client.Report("command", tags: new Dictionary { { "name", "home" } }); + client.Close(); + var again = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installIdFile: path); + again.Close(); + + Assert.True(IsGuid(id), id); + Assert.Equal(id, again.InstallId); + Assert.Equal(id + "\n", File.ReadAllText(path)); + Assert.Equal(new[] + { + "{\"application\":\"MyGame\",\"name\":\"startup\",\"tags\":{\"version\":\"1.2.3\",\"install\":\"" + id + "\"}}", + "{\"application\":\"MyGame\",\"name\":\"command\",\"tags\":{\"name\":\"home\",\"version\":\"1.2.3\",\"install\":\"" + id + "\"}}", + }, + _server.Received.Select(r => r.Body).ToArray()); + } + + [Fact] + public void Client_WithAnUnwritableFileStillReportsWithAnInMemoryId() + { + Directory.CreateDirectory(_directory); + string blocker = Path.Combine(_directory, "a-file"); + File.WriteAllText(blocker, "x"); + var log = new List(); + + var client = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", log: log.Add, + installIdFile: Path.Combine(blocker, "install-id")); + client.Report("startup"); + client.Close(); + + Assert.True(IsGuid(client.InstallId), client.InstallId); + Assert.Contains("\"install\":\"" + client.InstallId + "\"", _server.Received.Single().Body); + Assert.Contains(log, m => m.Contains("in-memory install ID")); + } + + [Fact] + public void DisabledClient_NeverMakesUpOrWritesAnId() + { + string path = Path.Combine(_directory, "install-id"); + var clients = new List + { + new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", enabled: false, installIdFile: path, installId: null), + new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", installIdFile: path), + new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", enabled: false, installId: "explicit"), + }; + _environment["DO_NOT_TRACK"] = "1"; + clients.Add(new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installIdFile: path)); + _environment.Clear(); + _environment["TRACE_USAGE_REPORTING"] = "off"; + clients.Add(new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installIdFile: path)); + + foreach (TraceClient client in clients) + { + Assert.False(client.IsEnabled); + Assert.Null(client.InstallId); + client.Report("startup"); + client.Close(); + } + + Assert.False(File.Exists(path)); + Assert.False(Directory.Exists(_directory)); + Thread.Sleep(200); + Assert.Empty(_server.Received); + } + + [Fact] + public void ExplicitId_IsTrimmedWinsOverTheFileAndWritesNothing() + { + string path = Path.Combine(_directory, "install-id"); + var client = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", + installId: " my-install ", installIdFile: path); + + client.Report("startup"); + client.Close(); + + Assert.Equal("my-install", client.InstallId); + Assert.False(File.Exists(path)); + Assert.Equal("{\"application\":\"MyGame\",\"name\":\"startup\",\"tags\":{\"version\":\"1.2.3\",\"install\":\"my-install\"}}", + _server.Received.Single().Body); + } + + [Fact] + public void NoIdOrABlankOne_SendsNoInstallTag() + { + var none = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k"); + var blank = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installId: " ", installIdFile: " "); + + none.Report("startup"); + blank.Report("startup"); + none.Close(); + blank.Close(); + + Assert.Null(none.InstallId); + Assert.Null(blank.InstallId); + Assert.All(_server.Received, r => Assert.DoesNotContain("install", r.Body)); + Assert.Equal(2, _server.Received.Count); + } + + [Fact] + public void ExplicitId_LongerThanTheLimitIsRejectedLikeAnOverlongVersion() + { + ArgumentException thrown = Assert.Throws( + () => new TraceClient("http://x", "MyGame", "1.2.3", key: "k", + installId: new string('i', TraceClient.MaxLength + 1))); + Assert.Equal("installId", thrown.ParamName); + // Rejected even for a disabled client: it is a programming error, not a runtime one. + Assert.Throws( + () => new TraceClient("http://x", "MyGame", "1.2.3", enabled: false, + installId: new string('i', TraceClient.MaxLength + 1))); + var atTheLimit = new TraceClient("http://x", "MyGame", "1.2.3", key: "k", + installId: " " + new string('i', TraceClient.MaxLength) + " "); + Assert.Equal(TraceClient.MaxLength, atTheLimit.InstallId.Length); + atTheLimit.Close(); + } + + [Fact] + public void Report_AnEventsOwnInstallTagWins() + { + var client = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installId: "configured"); + + client.Report("startup", tags: new Dictionary { { "install", "own" } }); + client.Close(); + + Assert.Equal("{\"application\":\"MyGame\",\"name\":\"startup\",\"tags\":{\"install\":\"own\",\"version\":\"1.2.3\"}}", + _server.Received.Single().Body); + } + + [Fact] + public void Report_NeverAddsTheInstallTagPastTheTagLimit() + { + var client = new TraceClient(_server.BaseUrl, "MyGame", "1.2.3", key: "k", installId: "configured"); + // With version, 31 own tags fill the limit: install is left off rather than the event dropped. + var full = Enumerable.Range(0, TraceClient.MaxTags - 1) + .Select(i => new KeyValuePair("t" + i, "v")).ToList(); + var roomForOne = full.Take(TraceClient.MaxTags - 2).ToList(); + + client.Report("full", tags: full); + client.Report("room", tags: roomForOne); + client.Close(); + + Dictionary bodies = _server.Received.ToDictionary(r => r.Body.Split('"')[7], r => r.Body); + Assert.DoesNotContain("install", bodies["full"]); + Assert.Contains("\"install\":\"configured\"", bodies["room"]); + } + + [Fact] + public void WithInstall_CountsOnlyTagsThatWouldBeSent() + { + var tags = Enumerable.Range(0, TraceClient.MaxTags - 1) + .Select(i => new KeyValuePair("t" + i, "v")).ToList(); + tags.Add(new KeyValuePair(" ", "skipped by Json")); + + List> merged = TraceClient.WithInstall(tags, "id"); + + Assert.Contains(new KeyValuePair("install", "id"), merged); + Assert.Same(tags, TraceClient.WithInstall(tags, null)); + } + } +}