diff --git a/Package.swift b/Package.swift index 98f46e77..078289c3 100644 --- a/Package.swift +++ b/Package.swift @@ -38,7 +38,10 @@ let package = Package( ), .testTarget( name: "XCLogParserTests", - dependencies: ["XCLogParser"] + dependencies: [ + "XCLogParser", + .product(name: "Gzip", package: "GzipSwift") + ] ), ] diff --git a/Package@swift-4.2.swift b/Package@swift-4.2.swift index 91b1f4bc..920bcbfc 100644 --- a/Package@swift-4.2.swift +++ b/Package@swift-4.2.swift @@ -31,7 +31,7 @@ let package = Package( ), .testTarget( name: "XCLogParserTests", - dependencies: ["XCLogParser"] + dependencies: ["XCLogParser", "Gzip"] ), ] diff --git a/Package@swift-5.0.swift b/Package@swift-5.0.swift index f5fe0663..0bdc35d9 100644 --- a/Package@swift-5.0.swift +++ b/Package@swift-5.0.swift @@ -34,7 +34,7 @@ let package = Package( ), .testTarget( name: "XCLogParserTests", - dependencies: ["XCLogParser"] + dependencies: ["XCLogParser", "Gzip"] ), ] diff --git a/Sources/XCLogParser/lexer/Lexer.swift b/Sources/XCLogParser/lexer/Lexer.swift index cbe52259..da86ebde 100644 --- a/Sources/XCLogParser/lexer/Lexer.swift +++ b/Sources/XCLogParser/lexer/Lexer.swift @@ -264,16 +264,6 @@ public final class Lexer { private extension Scanner { var approximateLine: String { - let currentLocation = self.offset - let contentSize = self.string.count - - let start = String.Index(compilerSafeOffset: currentLocation, in: self.string) - let endCount = contentSize - currentLocation > 21 ? currentLocation + 21 : contentSize - currentLocation - let end = String.Index(compilerSafeOffset: endCount, in: self.string) - - if end <= start { - return String(self.string[start..= self.stringEndIndex + offset >= bytes.count } init(string: String) { self.string = string + self.bytes = Array(string.utf8) self.offset = 0 } func scan(count: Int) -> String? { - let start = String.Index(compilerSafeOffset: self.offset, in: self.string) let endOffset = self.offset + count - guard endOffset <= self.string.utf16.count else { return nil } - - let end = String.Index(compilerSafeOffset: endOffset, in: self.string) - let result = self.string[start..= 0, + endOffset <= bytes.count, + let result = String(bytes: bytes[offset.. Bool { - guard self.string.starts(with: value) else { return false } + let valueBytes = Array(value.utf8) + let endOffset = offset + valueBytes.count + guard endOffset <= bytes.count, + bytes[offset..) -> String? { - var prefix: String = "" - var characterIndex = String.Index(compilerSafeOffset: self.offset, in: self.string) - - while characterIndex < self.stringEndIndex { - let character = self.string[characterIndex] + let allowedBytes = Set( + allowedCharacters.compactMap { character -> UInt8? in + let characterBytes = Array(String(character).utf8) + return characterBytes.count == 1 ? characterBytes[0] : nil + } + ) + var prefix: [UInt8] = [] - guard allowedCharacters.contains(character) else { + while !isAtEnd { + guard allowedBytes.contains(bytes[offset]) else { break } - prefix.append(character) + prefix.append(bytes[offset]) self.offset += 1 - characterIndex = String.Index(utf16Offset: self.offset, in: self.string) } - return prefix + return String(bytes: prefix, encoding: .utf8) } func moveOffset(by value: Int) { self.offset += value } + + func preview(count: Int) -> String { + let endOffset = min(offset + count, bytes.count) + return String(decoding: bytes[offset.. URL { + let url = directory.appendingPathComponent("test.xcactivitylog") + let data = contents.data(using: .utf8)! + try data.gzipped().write(to: url) + return url + } + + private func slfContents(from tokens: [Token]) throws -> String { + var classNames: [String] = [] + var contents = "SLF" + for token in tokens { + switch token { + case .int(let value): + contents += "\(value)#" + case .className(let value): + classNames.append(value) + contents += "\(value.utf8.count)%\(value)" + case .classNameRef(let value): + let index = try XCTUnwrap( + classNames.firstIndex(of: value), + "Class name \(value) must be emitted before it is referenced" + ) + contents += "\(index + 1)@" + case .string(let value): + contents += "\(value.utf8.count)\"\(value)" + case .double(let value): + contents += "\(String(value.bitPattern.byteSwapped, radix: 16))^" + case .null: + contents += "-" + case .list(let value): + contents += "\(value)(" + case .json(let value): + contents += "\(value.utf8.count)*\(value)" + } + } + return contents + } + } diff --git a/Tests/XCLogParserTests/LexerTests.swift b/Tests/XCLogParserTests/LexerTests.swift index 8760e163..8ebe9c99 100644 --- a/Tests/XCLogParserTests/LexerTests.swift +++ b/Tests/XCLogParserTests/LexerTests.swift @@ -56,6 +56,15 @@ class LexerTests: XCTestCase { XCTAssertEqual(stringToken, Token.string("Xcode.IDEActivityLogDomainType.BuildLog")) } + func testTokenizeStringUsesUTF8ByteLength() throws { + let value = "➜ Sources/Bundle+Locali🙂" + let logContents = "SLF0#\(value.utf8.count)\"\(value)1#" + + let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false) + + XCTAssertEqual(tokens, [.int(0), .string(value), .int(1)]) + } + func testTokenizeDouble() throws { let logContents = "SLF09#21%IDEActivityLogSection1@39\"Xcode.IDEActivityLogDomainType.BuildLog356098f239dfc041^" let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false) diff --git a/Tests/XCLogParserTests/LogLoaderTests.swift b/Tests/XCLogParserTests/LogLoaderTests.swift new file mode 100644 index 00000000..f1b17385 --- /dev/null +++ b/Tests/XCLogParserTests/LogLoaderTests.swift @@ -0,0 +1,21 @@ +import Foundation +import Gzip +import XCTest +@testable import XCLogParser + +class LogLoaderTests: XCTestCase { + + func testLoadFromURLPreservesEmbeddedNullBytes() throws { + let directory = try TestUtils.createRandomTestDir() + defer { try? FileManager.default.removeItem(at: directory) } + let url = directory.appendingPathComponent("test.xcactivitylog") + let value = "Sources/Bundle+Locali\u{0}zation" + let contents = "SLF0#\(value.utf8.count)\"\(value)1#" + let data = contents.data(using: .utf8)! + try data.gzipped().write(to: url) + + let loadedContents = try LogLoader().loadFromURL(url) + + XCTAssertEqual(loadedContents, contents) + } +} diff --git a/Tests/XCLogParserTests/XCTestManifests.swift b/Tests/XCLogParserTests/XCTestManifests.swift index 54d327a0..e18faedd 100644 --- a/Tests/XCLogParserTests/XCTestManifests.swift +++ b/Tests/XCLogParserTests/XCTestManifests.swift @@ -7,6 +7,7 @@ extension ActivityParserTests { // to regenerate. static let __allTests__ActivityParserTests = [ ("testParseActivityLog", testParseActivityLog), + ("testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes", testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes), ("testParseDBGConsoleLog", testParseDBGConsoleLog), ("testParseDVTTextDocumentLocation", testParseDVTTextDocumentLocation), ("testParseIBDocumentMemberLocation", testParseIBDocumentMemberLocation), @@ -65,10 +66,20 @@ extension LexerTests { ("testTokenizeListNil", testTokenizeListNil), ("testTokenizeString", testTokenizeString), ("testTokenizeStringRedacted", testTokenizeStringRedacted), + ("testTokenizeStringUsesUTF8ByteLength", testTokenizeStringUsesUTF8ByteLength), ("testTokenizeStringWithTokenDelimiters", testTokenizeStringWithTokenDelimiters), ] } +extension LogLoaderTests { + // DO NOT MODIFY: This is autogenerated, use: + // `swift test --generate-linuxmain` + // to regenerate. + static let __allTests__LogLoaderTests = [ + ("testLoadFromURLPreservesEmbeddedNullBytes", testLoadFromURLPreservesEmbeddedNullBytes), + ] +} + extension LogFinderTests { // DO NOT MODIFY: This is autogenerated, use: // `swift test --generate-linuxmain` @@ -138,6 +149,7 @@ public func __allTests() -> [XCTestCaseEntry] { testCase(IssuesReporterTests.__allTests__IssuesReporterTests), testCase(LexRedactorTests.__allTests__LexRedactorTests), testCase(LexerTests.__allTests__LexerTests), + testCase(LogLoaderTests.__allTests__LogLoaderTests), testCase(LogFinderTests.__allTests__LogFinderTests), testCase(LogManifestTests.__allTests__LogManifestTests), testCase(ParserTests.__allTests__ParserTests),