Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
697 changes: 697 additions & 0 deletions data/10.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,226 changes: 1,226 additions & 0 deletions data/10.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

735 changes: 735 additions & 0 deletions data/11.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,273 changes: 1,273 additions & 0 deletions data/11.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

755 changes: 755 additions & 0 deletions data/12.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,286 changes: 1,286 additions & 0 deletions data/12.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

755 changes: 755 additions & 0 deletions data/12.1.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,286 changes: 1,286 additions & 0 deletions data/12.1.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

790 changes: 790 additions & 0 deletions data/13.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,310 changes: 1,310 additions & 0 deletions data/13.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

796 changes: 796 additions & 0 deletions data/14.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,320 changes: 1,320 additions & 0 deletions data/14.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

808 changes: 808 additions & 0 deletions data/15.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,347 changes: 1,347 additions & 0 deletions data/15.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

808 changes: 808 additions & 0 deletions data/15.1.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,347 changes: 1,347 additions & 0 deletions data/15.1.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

836 changes: 836 additions & 0 deletions data/16.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,400 changes: 1,400 additions & 0 deletions data/16.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

842 changes: 842 additions & 0 deletions data/17.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,419 changes: 1,419 additions & 0 deletions data/17.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

418 changes: 418 additions & 0 deletions data/6.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

757 changes: 757 additions & 0 deletions data/6.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

446 changes: 446 additions & 0 deletions data/6.1.0-indic-positional-category.txt

Large diffs are not rendered by default.

796 changes: 796 additions & 0 deletions data/6.1.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

446 changes: 446 additions & 0 deletions data/6.2.0-indic-positional-category.txt

Large diffs are not rendered by default.

796 changes: 796 additions & 0 deletions data/6.2.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

446 changes: 446 additions & 0 deletions data/6.3.0-indic-positional-category.txt

Large diffs are not rendered by default.

796 changes: 796 additions & 0 deletions data/6.3.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

501 changes: 501 additions & 0 deletions data/7.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,042 changes: 1,042 additions & 0 deletions data/7.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

632 changes: 632 additions & 0 deletions data/8.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,121 changes: 1,121 additions & 0 deletions data/8.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

663 changes: 663 additions & 0 deletions data/9.0.0-indic-positional-category.txt

Large diffs are not rendered by default.

1,159 changes: 1,159 additions & 0 deletions data/9.0.0-indic-syllabic-category.txt

Large diffs are not rendered by default.

34 changes: 34 additions & 0 deletions data/resources.js
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ const resources = [
'case-folding': 'https://unicode.org/Public/6.0.0/ucd/CaseFolding.txt',
'special-casing': 'https://unicode.org/Public/6.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/6.0.0/ucd/BidiMirroring.txt',
'indic-positional-category': 'https://unicode.org/Public/6.0.0/ucd/IndicMatraCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/6.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/6.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/6.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -315,6 +317,8 @@ const resources = [
'case-folding': 'https://unicode.org/Public/6.1.0/ucd/CaseFolding.txt',
'special-casing': 'https://unicode.org/Public/6.1.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/6.1.0/ucd/BidiMirroring.txt',
'indic-positional-category': 'https://unicode.org/Public/6.1.0/ucd/IndicMatraCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/6.1.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/6.1.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/6.1.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -345,6 +349,8 @@ const resources = [
'case-folding': 'https://unicode.org/Public/6.2.0/ucd/CaseFolding.txt',
'special-casing': 'https://unicode.org/Public/6.2.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/6.2.0/ucd/BidiMirroring.txt',
'indic-positional-category': 'https://unicode.org/Public/6.2.0/ucd/IndicMatraCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/6.2.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/6.2.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/6.2.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -376,6 +382,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/6.3.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/6.3.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/6.3.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/6.3.0/ucd/IndicMatraCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/6.3.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/6.3.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/6.3.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -405,6 +413,8 @@ const resources = [
'https://unicode.org/Public/7.0.0/ucd/CompositionExclusions.txt',
'case-folding': 'https://unicode.org/Public/7.0.0/ucd/CaseFolding.txt',
'special-casing': 'https://unicode.org/Public/7.0.0/ucd/SpecialCasing.txt',
'indic-positional-category': 'https://unicode.org/Public/7.0.0/ucd/IndicMatraCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/7.0.0/ucd/IndicSyllabicCategory.txt',
'bidi-mirroring': 'https://unicode.org/Public/7.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/7.0.0/ucd/BidiBrackets.txt',
'line-break': 'https://unicode.org/Public/7.0.0/ucd/LineBreak.txt',
Expand Down Expand Up @@ -438,6 +448,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/8.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/8.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/8.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/8.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/8.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/8.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/8.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -469,6 +481,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/9.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/9.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/9.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/9.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/9.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/9.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/9.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -500,6 +514,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/10.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/10.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/10.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/10.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/10.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/10.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/10.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -531,6 +547,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/11.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/11.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/11.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/11.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/11.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/11.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/11.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -572,6 +590,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/12.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/12.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/12.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/12.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/12.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/12.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/12.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -613,6 +633,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/12.1.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/12.1.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/12.1.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/12.1.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/12.1.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/12.1.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/12.1.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -654,6 +676,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/13.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/13.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/13.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/13.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/13.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/13.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/13.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -695,6 +719,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/14.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/14.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/14.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/14.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/14.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/14.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/14.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -736,6 +762,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/15.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/15.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/15.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/15.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/15.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/15.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/15.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -777,6 +805,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/15.1.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/15.1.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/15.1.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/15.1.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/15.1.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/15.1.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/15.1.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -818,6 +848,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/16.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/16.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/16.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/16.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/16.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/16.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/16.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down Expand Up @@ -859,6 +891,8 @@ const resources = [
'special-casing': 'https://unicode.org/Public/17.0.0/ucd/SpecialCasing.txt',
'bidi-mirroring': 'https://unicode.org/Public/17.0.0/ucd/BidiMirroring.txt',
'bidi-brackets': 'https://unicode.org/Public/17.0.0/ucd/BidiBrackets.txt',
'indic-positional-category': 'https://unicode.org/Public/17.0.0/ucd/IndicPositionalCategory.txt',
'indic-syllabic-category': 'https://unicode.org/Public/17.0.0/ucd/IndicSyllabicCategory.txt',
'line-break': 'https://unicode.org/Public/17.0.0/ucd/LineBreak.txt',
'grapheme-cluster-break':
'https://unicode.org/Public/17.0.0/ucd/auxiliary/GraphemeBreakProperty.txt',
Expand Down
17 changes: 17 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ parsers.parseBidiBrackets = require('./scripts/parse-bidi-brackets.js');
parsers.parseCaseFolding = require('./scripts/parse-case-folding.js');
parsers.parseBidiClass = require('./scripts/parse-bidi-class.js');
parsers.parseCompositionExclusions = require('./scripts/parse-composition-exclusions.js');
parsers.parseIndicPositionalCategory = require('./scripts/parse-indic-positional-category.js');
parsers.parseIndicSyllabicCategory = require('./scripts/parse-indic-syllabic-category.js');
parsers.parseLineBreak = require('./scripts/parse-line-break.js');
parsers.parseScriptExtensions = require('./scripts/parse-script-extensions.js');
parsers.parseSpecialCasing = require('./scripts/parse-special-casing.js');
Expand Down Expand Up @@ -120,6 +122,21 @@ const generateData = function(version) {
'map': parsers.parseBidiBrackets(version),
'type': 'Bidi_Paired_Bracket_Type'
}));
{
const InPCName = +version.split(".")[0] >= 8 ? 'Indic_Positional_Category' : 'Indic_Matra_Category';
console.log('Parsing Unicode v%s `%s`…', version, InPCName);
extend(dirMap, utils.writeFiles({
'version': version,
'map': parsers.parseIndicPositionalCategory(version),
'type': InPCName
}));
}
console.log('Parsing Unicode v%s `Indic_Syllabic_Category`…', version);
extend(dirMap, utils.writeFiles({
'version': version,
'map': parsers.parseIndicSyllabicCategory(version),
'type': 'Indic_Syllabic_Category'
}));
console.log('Parsing Unicode v%s `Line_Break`…', version);
extend(dirMap, utils.writeFiles({
'version': version,
Expand Down
2 changes: 2 additions & 0 deletions scripts/download.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ const TYPES = [
'special-casing',
'bidi-mirroring',
'bidi-brackets',
'indic-positional-category',
'indic-syllabic-category',
'line-break',
'grapheme-cluster-break',
'word-break',
Expand Down
43 changes: 43 additions & 0 deletions scripts/parse-indic-positional-category.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
'use strict';

const utils = require('./utils.js');
const regenerate = require('regenerate');

const parseIndicPositionalCategory = function(version) {
const source = utils.readDataFile(version, 'indic-positional-category');
if (!source) {
return;
}
const NAKey = +version.split(".")[0] >= 17 ? 'Not_Applicable' : 'NA';
const map = {
// All code points not explicitly listed for Indic_Positional_Category have the value Not_Applicable (NA).
[NAKey]: regenerate().addRange(0, 0x10FFFF)
};
const lines = source.split('\n');
for (const line of lines) {
if (!line || /^#/.test(line)) {
continue;
}
const data = line.trim().split(';');
const charRange = data[0].replace('..', '-').trim();
const rangeParts = charRange.split('-');
const value = data[1].split('#')[0].trim();
const propertyValue = value;
map[propertyValue] ??= regenerate();
if (rangeParts.length == 2) {
const [from, to] = [
parseInt(rangeParts[0], 16),
parseInt(rangeParts[1], 16),
];
map[NAKey].removeRange(from, to);
map[propertyValue].addRange(from, to);
} else {
const codePoint = parseInt(charRange, 16);
map[NAKey].remove(codePoint);
map[propertyValue].add(codePoint);
}
}
return map;
};

module.exports = parseIndicPositionalCategory;
42 changes: 42 additions & 0 deletions scripts/parse-indic-syllabic-category.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
'use strict';

const utils = require('./utils.js');
const regenerate = require('regenerate');

const parseIndicSyllabicCategory = function(version) {
const source = utils.readDataFile(version, 'indic-syllabic-category');
if (!source) {
return;
}
const map = {
// All code points not explicitly listed for Indic_Syllabic_Category have the value Other.
'Other': regenerate().addRange(0, 0x10FFFF)
};
const lines = source.split('\n');
for (const line of lines) {
if (!line || /^#/.test(line)) {
continue;
}
const data = line.trim().split(';');
const charRange = data[0].replace('..', '-').trim();
const rangeParts = charRange.split('-');
const value = data[1].split('#')[0].trim();
const propertyValue = value;
map[propertyValue] ??= regenerate();
if (rangeParts.length == 2) {
const [from, to] = [
parseInt(rangeParts[0], 16),
parseInt(rangeParts[1], 16),
];
map['Other'].removeRange(from, to);
map[propertyValue].addRange(from, to);
} else {
const codePoint = parseInt(charRange, 16);
map['Other'].remove(codePoint);
map[propertyValue].add(codePoint);
}
}
return map;
};

module.exports = parseIndicSyllabicCategory;
Loading