Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,14 @@ Deploy the updated **read-only** `TraceParserMCP/dab-config.json` as part of thi

### Deletion regression checks

The dependency-free console harness checks denied anonymous/wrong-tenant calls, ordinary tenant-user access, disabled configuration, incremental completion, failure/cancellation propagation, and read-only DAB permissions. It uses synthetic identities and a fake store, with no database or network calls:
The dependency-free console harness checks deletion authorization/completion, read-only DAB permissions, import-status query/error handling, visible deletion feedback, and non-overlapping status polls. It uses synthetic identities and fake stores/HTTP responses, with no database or network calls:

```powershell
dotnet run --project .\tests\TraceParserWeb.RegressionTests -c Release
```

## Import status and database maintenance

Status polling uses DAB's `$first` parameter, not OData's unsupported `$top`. The upload and trace-list pages share the same database-milestone checks. HTTP failures, timeouts and malformed responses are displayed as **status unavailable**, not as evidence that the Function is parsing or queued. Each page permits only one status poll at a time.

Trace-list labels describe data availability, not a live worker heartbeat. For example, an empty trace has no parsed session data; that alone does not prove an ETL job is running. SQL index maintenance can delay status queries and deletion. Deletion errors appear above the trace list so a timeout is not hidden below other records. Do not treat a timeout as successful deletion or repeatedly submit deletes while maintenance is blocking SQL.
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@
</div>
</div>

@if (!string.IsNullOrEmpty(_statusMessage))
{
<div role="status" class="traces-status @(_statusIsError ? "status-error" : "status-success")">
@_statusMessage
</div>
}

@if (_loading)
{
<div class="traces-loading">
Expand Down Expand Up @@ -89,7 +96,6 @@
else if (_importingStages.TryGetValue(trace.TraceId, out var stage))
{
<div class="trace-importing">
<span class="importing-spinner"></span>
<span class="importing-text">@GetImportStageText(stage)</span>
</div>
}
Expand Down Expand Up @@ -126,12 +132,6 @@
</div>
}

@if (!string.IsNullOrEmpty(_statusMessage))
{
<div class="traces-status @(_statusIsError ? "status-error" : "status-success")">
@_statusMessage
</div>
}
</div>
</div>

Expand All @@ -147,6 +147,8 @@
string? _statusMessage;
bool _statusIsError;
System.Threading.Timer? _importPollTimer;
int _pollInProgress;
bool _disposed;

protected override async Task OnInitializedAsync()
{
Expand Down Expand Up @@ -193,28 +195,40 @@
// Check stage for each trace without stats
foreach (var trace in tracesWithoutStats)
{
if (_disposed) return;
var stage = await TraceSvc.GetImportStageAsync(trace.TraceId);
if (_disposed) return;
if (stage != ImportStage.Complete)
_importingStages[trace.TraceId] = stage;
}

if (_importingStages.Count == 0) return;

// Start polling importing traces every 10s
_importPollTimer = new System.Threading.Timer(async _ =>
_importPollTimer = new System.Threading.Timer(async _ => await PollImportStagesAsync(),
null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
}

async Task PollImportStagesAsync()
{
if (Interlocked.CompareExchange(ref _pollInProgress, 1, 0) != 0) return;
try
{
try
await InvokeAsync(async () =>
{
if (_disposed) return;
var changed = false;
foreach (var traceId in _importingStages.Keys.ToList())
{
var stage = await TraceSvc.GetImportStageAsync(traceId);
if (_disposed) return;
if (!_importingStages.TryGetValue(traceId, out var previousStage)) continue;
if (stage == ImportStage.Complete)
{
_importingStages.Remove(traceId);
changed = true;
}
else if (_importingStages[traceId] != stage)
else if (previousStage != stage)
{
_importingStages[traceId] = stage;
changed = true;
Expand All @@ -236,13 +250,17 @@
}
}

await InvokeAsync(StateHasChanged);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Import poll error on Traces page");
}
}, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
if (!_disposed) StateHasChanged();
});
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Import poll error on Traces page");
}
finally
{
Volatile.Write(ref _pollInProgress, 0);
}
}

void RequestDelete(int traceId)
Expand Down Expand Up @@ -270,6 +288,7 @@
_statusIsError = false;
_traces.RemoveAll(t => t.TraceId == traceId);
_traceStats.Remove(traceId);
_importingStages.Remove(traceId);
}
catch (Exception ex)
{
Expand Down Expand Up @@ -304,14 +323,16 @@
static string GetImportStageText(ImportStage stage) => stage switch
{
ImportStage.WaitingForFunction => "Waiting for processing...",
ImportStage.Parsing => "Parsing ETL events...",
ImportStage.ProcessingDimensions => "Importing trace lines...",
ImportStage.Finalizing => "Finalizing...",
_ => "Importing..."
ImportStage.Parsing => "No parsed session data available yet.",
ImportStage.ProcessingDimensions => "Waiting for trace lines.",
ImportStage.Finalizing => "Waiting for session metrics.",
ImportStage.Unavailable => "Import status unavailable; the next poll will retry.",
_ => "Import status not confirmed."
};

public void Dispose()
{
_disposed = true;
_importPollTimer?.Dispose();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@
long _uploadedBytes;
string StatusMessage = "";
System.Threading.Timer? _pollTimer;
int _pollInProgress;
bool _disposed;
DotNetObjectReference<EtlUpload>? _dotNetRef;
ImportStage _currentStage = ImportStage.WaitingForFunction;
DateTime _processingStartTime;
Expand Down Expand Up @@ -278,11 +280,20 @@
Logger.LogInformation("Direct blob upload complete for session {Session}", SessionName);

// Start polling for import stage
_pollTimer = new System.Threading.Timer(async _ =>
_pollTimer = new System.Threading.Timer(async _ => await PollImportStatusAsync(),
null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
}

async Task PollImportStatusAsync()
{
if (Interlocked.CompareExchange(ref _pollInProgress, 1, 0) != 0) return;
try
{
try
await InvokeAsync(async () =>
{
if (_disposed) return;
var status = await UploadSvc.GetImportStatusAsync(SessionName, CancellationToken.None);
if (_disposed) return;

if (status.Stage != _currentStage)
{
Expand All @@ -300,13 +311,17 @@
StatusMessage = $"Import complete! Session '{SessionName}' is ready to query in Chat.";
}

await InvokeAsync(StateHasChanged);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Status poll error for session {Session}", SessionName);
}
}, null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
StateHasChanged();
});
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Status poll error for session {Session}", SessionName);
}
finally
{
Volatile.Write(ref _pollInProgress, 0);
}
}

[JSInvokable]
Expand Down Expand Up @@ -339,6 +354,7 @@
ImportStage.ProcessingDimensions => "Promoting trace lines and rebuilding indexes...",
ImportStage.Finalizing => "Computing session aggregations...",
ImportStage.Complete => "Import complete!",
ImportStage.Unavailable => "Import status unavailable; the next poll will retry.",
_ => "Processing..."
};

Expand All @@ -348,6 +364,7 @@
ImportStage.Parsing => "Parsing",
ImportStage.ProcessingDimensions => "Importing",
ImportStage.Finalizing => "Finalizing",
ImportStage.Unavailable => "Unavailable",
_ => ""
};

Expand Down Expand Up @@ -384,6 +401,7 @@

public void Dispose()
{
_disposed = true;
_pollTimer?.Dispose();
_dotNetRef?.Dispose();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace TraceParserWeb.Services;

public enum ImportStage
{
Unavailable = -1,
WaitingForFunction,
Parsing,
ProcessingDimensions,
Expand All @@ -29,7 +30,11 @@ public class EtlImportOptions
public string DabBaseUrl { get; set; } = "";
}

public class EtlUploadService(IOptions<EtlImportOptions> opts, IHttpClientFactory httpFactory)
public class EtlUploadService(
IOptions<EtlImportOptions> opts,
IHttpClientFactory httpFactory,
TraceService traceService,
ILogger<EtlUploadService> logger)
{
/// <summary>
/// Uploads an ETL file to Azure Blob Storage under {sessionName}/{fileName}.
Expand Down Expand Up @@ -80,40 +85,29 @@ public async Task<ImportStatus> GetImportStatusAsync(string sessionName, Cancell
{
try
{
var http = httpFactory.CreateClient("dab");
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
cts.CancelAfter(TimeSpan.FromSeconds(10));
using var http = httpFactory.CreateClient("dab");

// Step 1: Find the trace
var url = $"/api/Traces?$filter=TraceName eq '{Uri.EscapeDataString(sessionName)}'";
var resp = await http.GetFromJsonAsync<JsonElement>(url, ct);
if (!resp.TryGetProperty("value", out var arr) || arr.GetArrayLength() == 0)
var escapedName = Uri.EscapeDataString(sessionName.Replace("'", "''"));
var url = $"/api/Traces?$filter=TraceName eq '{escapedName}'&$first=1&$select=TraceId";
var resp = await http.GetFromJsonAsync<JsonElement>(url, cts.Token);
var arr = TraceService.ReadRows(resp);
if (arr.GetArrayLength() == 0)
return new ImportStatus { Stage = ImportStage.WaitingForFunction };

var traceId = arr[0].GetProperty("TraceId").GetInt32();

// Step 2: Check threads exist (created during BulkInsertDimensions, after staging)
var threadUrl = $"/api/UserSessionProcessThreads?$filter=TraceId eq {traceId}&$top=1";
var threadResp = await http.GetFromJsonAsync<JsonElement>(threadUrl, ct);
if (!threadResp.TryGetProperty("value", out var threadArr) || threadArr.GetArrayLength() == 0)
return new ImportStatus { Stage = ImportStage.Parsing, TraceId = traceId };

// Step 3: Verify TraceLines exist (the final promote step)
var threadId = threadArr[0].GetProperty("UserSessionProcessThreadId").GetInt32();
var tlUrl = $"/api/TraceLines?$filter=UserSessionProcessThreadId eq {threadId}&$top=1";
var tlResp = await http.GetFromJsonAsync<JsonElement>(tlUrl, ct);
if (!tlResp.TryGetProperty("value", out var tlArr) || tlArr.GetArrayLength() == 0)
return new ImportStatus { Stage = ImportStage.ProcessingDimensions, TraceId = traceId };

// Step 4: Check SessionMetrics exist (aggregation complete)
var smUrl = $"/api/SessionMetrics?$filter=TraceId eq {traceId}&$top=1";
var smResp = await http.GetFromJsonAsync<JsonElement>(smUrl, ct);
if (!smResp.TryGetProperty("value", out var smArr) || smArr.GetArrayLength() == 0)
return new ImportStatus { Stage = ImportStage.Finalizing, TraceId = traceId };

return new ImportStatus { Stage = ImportStage.Complete, TraceId = traceId };
var traceId = TraceService.ReadId(arr[0], "TraceId");
return new ImportStatus {
Stage = await traceService.GetImportStageAsync(traceId, cts.Token),
TraceId = traceId
};
}
catch
catch (Exception ex) when (ex is HttpRequestException or JsonException or OperationCanceledException)
{
return new ImportStatus { Stage = ImportStage.WaitingForFunction };
ct.ThrowIfCancellationRequested();
logger.LogWarning(ex, "Import status lookup unavailable");
return new ImportStatus { Stage = ImportStage.Unavailable };
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public class TraceStats
public int TotalDatabaseCalls { get; set; }
}

public class TraceService(IHttpClientFactory httpFactory)
public class TraceService(IHttpClientFactory httpFactory, ILogger<TraceService> logger)
{
public async Task<List<TraceDto>> GetTracesAsync(CancellationToken ct = default)
{
Expand Down Expand Up @@ -110,32 +110,53 @@ public async Task<ImportStage> GetImportStageAsync(int traceId, CancellationToke
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
cts.CancelAfter(TimeSpan.FromSeconds(10));

var http = httpFactory.CreateClient("dab");
using var http = httpFactory.CreateClient("dab");

// Check USPT exists
var threadUrl = $"/api/UserSessionProcessThreads?$filter=TraceId eq {traceId}&$top=1";
var threadUrl = $"/api/UserSessionProcessThreads?$filter=TraceId eq {traceId}&$first=1&$select=UserSessionProcessThreadId";
var threadResp = await http.GetFromJsonAsync<JsonElement>(threadUrl, cts.Token);
if (!threadResp.TryGetProperty("value", out var threadArr) || threadArr.GetArrayLength() == 0)
var threadArr = ReadRows(threadResp);
if (threadArr.GetArrayLength() == 0)
return ImportStage.Parsing;

// Check TraceLines exist
var threadId = threadArr[0].GetProperty("UserSessionProcessThreadId").GetInt32();
var tlUrl = $"/api/TraceLines?$filter=UserSessionProcessThreadId eq {threadId}&$top=1";
var threadId = ReadId(threadArr[0], "UserSessionProcessThreadId");
var tlUrl = $"/api/TraceLines?$filter=UserSessionProcessThreadId eq {threadId}&$first=1&$select=TraceLineId";
var tlResp = await http.GetFromJsonAsync<JsonElement>(tlUrl, cts.Token);
if (!tlResp.TryGetProperty("value", out var tlArr) || tlArr.GetArrayLength() == 0)
if (ReadRows(tlResp).GetArrayLength() == 0)
return ImportStage.ProcessingDimensions;

// Check SessionMetrics exist
var smUrl = $"/api/SessionMetrics?$filter=TraceId eq {traceId}&$top=1";
var smUrl = $"/api/SessionMetrics?$filter=TraceId eq {traceId}&$first=1&$select=TraceId";
var smResp = await http.GetFromJsonAsync<JsonElement>(smUrl, cts.Token);
if (!smResp.TryGetProperty("value", out var smArr) || smArr.GetArrayLength() == 0)
if (ReadRows(smResp).GetArrayLength() == 0)
return ImportStage.Finalizing;

return ImportStage.Complete;
}
catch
catch (Exception ex) when (ex is HttpRequestException or JsonException or OperationCanceledException)
{
return ImportStage.Parsing;
ct.ThrowIfCancellationRequested();
logger.LogWarning(ex, "Import status unavailable for trace {TraceId}", traceId);
return ImportStage.Unavailable;
}
}

internal static JsonElement ReadRows(JsonElement response)
{
if (response.ValueKind != JsonValueKind.Object
|| !response.TryGetProperty("value", out var rows)
|| rows.ValueKind != JsonValueKind.Array)
throw new JsonException("DAB returned an invalid row collection.");
return rows;
}

internal static int ReadId(JsonElement row, string property)
{
if (row.ValueKind != JsonValueKind.Object
|| !row.TryGetProperty(property, out var id)
|| id.ValueKind != JsonValueKind.Number || !id.TryGetInt32(out var value))
throw new JsonException("DAB returned an invalid identifier.");
return value;
}
}
Loading