Skip to content

Commit

Permalink
add batch localization tool
Browse files Browse the repository at this point in the history
  • Loading branch information
ArthurHeitmann committed Jan 23, 2025
1 parent 2028c81 commit 2ba3f06
Show file tree
Hide file tree
Showing 16 changed files with 1,622 additions and 173 deletions.
10 changes: 10 additions & 0 deletions lib/fileTypeUtils/dat/datExtractor.dart
Original file line number Diff line number Diff line change
Expand Up @@ -131,3 +131,13 @@ Stream<ExtractedInnerFile> extractDatFilesAsStream(String datPath) async* {
return;
}
}

Future<List<String>> peekDatFileNames(String datPath) async {
var bytes = await ByteDataWrapper.fromFile(datPath);
var header = _DatHeader(bytes);
bytes.position = header.fileNamesOffset;
var nameLength = bytes.readUint32();
return List<String>
.generate(header.fileNumber, (index) =>
bytes.readString(nameLength).split("\u0000")[0]);
}
5 changes: 4 additions & 1 deletion lib/fileTypeUtils/smd/smdReader.dart
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@

import 'dart:math';

import '../utils/ByteDataWrapper.dart';

class SmdEntry {
Expand All @@ -21,7 +23,8 @@ Future<List<SmdEntry>> readSmdFile(String path) async {
for (int i = 0; i < count; i++) {
String id = reader.readString(0x80, encoding: StringEncoding.utf16);
int indexX10 = reader.readUint64();
String text = reader.readString(0x800, encoding: StringEncoding.utf16);
var textSize = min(0x800, reader.length - reader.position);
String text = reader.readString(textSize, encoding: StringEncoding.utf16);
var zerosRemover = RegExp("\x00+\$");
id = id.replaceAll(zerosRemover, "");
text = text.replaceAll(zerosRemover, "");
Expand Down
2 changes: 2 additions & 0 deletions lib/fileTypeUtils/smd/smdWriter.dart
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ Future<void> saveSmd(List<SmdEntry> entries, String path) async {
for (var entry in entries) {
var id = entry.id.padRight(0x40, '\x00');
var text = entry.text.padRight(0x400, '\x00');
if (text.contains("\n") && !text.contains("\r\n"))
text = text.replaceAll("\n", "\r\n");
bytes.writeString(id, StringEncoding.utf16);
bytes.writeUint64(entry.indexX10);
bytes.writeString(text, StringEncoding.utf16);
Expand Down
3 changes: 3 additions & 0 deletions lib/stateManagement/openFiles/openFileTypes.dart
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import '../undoable.dart';
import 'types/BnkFilePlaylistData.dart';
import 'types/BxmFileData.dart';
import 'types/EstFileData.dart';
import 'types/FontSettingsDummy.dart';
import 'types/FtbFileData.dart';
import 'types/McdFileData.dart';
import 'types/RubyFileData.dart';
Expand Down Expand Up @@ -100,6 +101,8 @@ abstract class OpenFileData with HasUuid, Undoable, Disposable, HasUndoHistory {
return EstFileData(name, path, secondaryName: secondaryName);
else if (path == "preferences")
return PreferencesData();
else if (path == "fontSettings")
return FontSettingsDummy();
else
return TextFileData(name, path, secondaryName: secondaryName);
}
Expand Down
24 changes: 24 additions & 0 deletions lib/stateManagement/openFiles/types/FontSettingsDummy.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@

import 'package:flutter/material.dart';

import '../../../widgets/filesView/FileType.dart';
import '../../undoable.dart';
import '../openFileTypes.dart';
import 'McdFileData.dart';

class FontSettingsDummy extends OpenFileData {
FontSettingsDummy() : super("Font Settings", "", type: FileType.fontSettings, icon: Icons.text_fields) {
canBeReloaded = false;
if (McdData.availableFonts.isEmpty)
McdData.loadAvailableFonts();
}

@override
void restoreWith(Undoable snapshot) {
}

@override
Undoable takeSnapshot() {
return FontSettingsDummy();
}
}
36 changes: 22 additions & 14 deletions lib/stateManagement/openFiles/types/McdFileData.dart
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ class McdFileData extends OpenFileData {
}

abstract class _McdFilePart with HasUuid, Undoable implements Disposable {
OpenFileId file;
OpenFileId? file;

_McdFilePart(this.file);

Expand Down Expand Up @@ -436,13 +436,15 @@ class McdParagraph extends _McdFilePart {

void addLine() {
lines.add(McdLine(file, StringProp("", fileId: file)));
areasManager.onFileIdUndoEvent(file);
if (file != null)
areasManager.onFileIdUndoEvent(file!);
}

void removeLine(int index) {
lines.removeAt(index)
.dispose();
areasManager.onFileIdUndoEvent(file);
if (file != null)
areasManager.onFileIdUndoEvent(file!);
}

@override
Expand Down Expand Up @@ -526,13 +528,15 @@ class McdEvent extends _McdFilePart {
NumberProp(fontId, true, fileId: file),
ValueListNotifier([], fileId: file)
));
areasManager.onFileIdUndoEvent(file);
if (file != null)
areasManager.onFileIdUndoEvent(file!);
}

void removeParagraph(int index) {
paragraphs.removeAt(index)
.dispose();
areasManager.onFileIdUndoEvent(file);
if (file != null)
areasManager.onFileIdUndoEvent(file!);
}

@override
Expand Down Expand Up @@ -593,13 +597,15 @@ class McdData extends _McdFilePart {
static NumberProp fontAtlasResolutionScale = NumberProp(1.0, false, fileId: null);
static ChangeNotifier fontChanges = ChangeNotifier();

final String mcdPath;
final StringProp textureWtaPath;
final StringProp textureWtpPath;
final int firstMsgSeqNum;
ValueListNotifier<McdEvent> events;
Map<int, McdLocalFont> usedFonts;
Future<void> Function(String) exportDatFunc = exportDat;

McdData(super.file, this.textureWtaPath, this.textureWtpPath, this.usedFonts, this.firstMsgSeqNum, this.events) {
McdData(super.file, this.mcdPath, this.textureWtaPath, this.textureWtpPath, this.usedFonts, this.firstMsgSeqNum, this.events) {
events.addListener(onDataChanged);
}

Expand Down Expand Up @@ -628,7 +634,7 @@ class McdData extends _McdFilePart {
return null;
}

static Future<McdData> fromMcdFile(OpenFileId file, String mcdPath) async {
static Future<McdData> fromMcdFile(OpenFileId? file, String mcdPath) async {
var datDir = dirname(mcdPath);
var mcdName = basenameWithoutExtension(mcdPath);
String? wtpPath = await searchForTexFile(datDir, mcdName, ".wtp");
Expand Down Expand Up @@ -665,6 +671,7 @@ class McdData extends _McdFilePart {

return McdData(
file,
mcdPath,
StringProp(wtaPath, fileId: file),
StringProp(wtpPath, fileId: file),
{for (var f in usedFonts) f.fontId: f},
Expand Down Expand Up @@ -705,13 +712,15 @@ class McdData extends _McdFilePart {
StringProp("NEW_EVENT_NAME${suffix.isNotEmpty ? "_$suffix" : ""}", fileId: file),
ValueListNotifier([], fileId: file)
));
areasManager.onFileIdUndoEvent(file);
if (file != null)
areasManager.onFileIdUndoEvent(file!);
}

void removeEvent(int index) {
events.removeAt(index)
.dispose();
areasManager.onFileIdUndoEvent(file);
if (file != null)
areasManager.onFileIdUndoEvent(file!);
}

static void addFontOverride() {
Expand Down Expand Up @@ -907,11 +916,9 @@ class McdData extends _McdFilePart {
exportEvents.sort((a, b) => a.id.compareTo(b.id));

var mcdFile = McdFile.fromParts(header, exportMessages, exportSymbols, exportGlyphs, exportFonts, exportEvents);
var openFile = areasManager.fromId(file)!;
await mcdFile.writeToFile(openFile.path);
await mcdFile.writeToFile(mcdPath);

print("Saved MCD file");
messageLog.add("Saved MCD file ${basename(openFile.path)}");
messageLog.add("Saved MCD file ${basename(mcdPath)}");
}

Future<void> updateFontsTexture() async {
Expand Down Expand Up @@ -1162,7 +1169,7 @@ class McdData extends _McdFilePart {

// export dtt
var dttPath = dirname(textureWtpPath.value);
await exportDat(dttPath);
await exportDatFunc(dttPath);

print("Generated font texture with ${symbols.length} symbols");

Expand Down Expand Up @@ -1223,6 +1230,7 @@ class McdData extends _McdFilePart {
Undoable takeSnapshot() {
var snapshot = McdData(
file,
mcdPath,
textureWtaPath.takeSnapshot() as StringProp,
textureWtpPath.takeSnapshot() as StringProp,
usedFonts.map((id, font) => MapEntry(id, font)),
Expand Down
182 changes: 182 additions & 0 deletions lib/utils/batchLocalization/BatchLocalizationData.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@

enum BatchLocalizationLanguage {
jp, de, es, fr, it, us, kor, cn;

static BatchLocalizationLanguage fromString(String value) {
for (final language in values) {
if (language.name == value) {
return language;
}
}
throw Exception("Invalid language: $value");
}
}

class BatchLocalizationData {
final List<BatchLocalizationFileData> files;
final BatchLocalizationLanguage language;

BatchLocalizationData(this.files, this.language);

factory BatchLocalizationData.read(StringReader reader) {
var langKey = reader.readUntil(_listSeparator);
assert(langKey == "Original Language");
final langStr = reader.readUntil("\n$_fileSeparator");
final language = BatchLocalizationLanguage.fromString(langStr);
final files = <BatchLocalizationFileData>[];
while (!reader.isEOF) {
files.add(BatchLocalizationFileData.read(reader));
}
return BatchLocalizationData(files, language);
}

factory BatchLocalizationData.fromJson(Map json) {
final language = BatchLocalizationLanguage.fromString(json["originalLanguage"]);
final files = (json["files"] as List).map((e) => BatchLocalizationFileData.fromJson(e)).toList();
return BatchLocalizationData(files, language);
}

void writeString(StringSink sink) {
sink.write("Original Language");
sink.write(_listSeparator);
sink.write(language.name);
sink.writeln();
sink.write(_fileSeparator);
for (final file in files) {
file.writeString(sink);
}
}

Map<String, dynamic> toJson() {
return {
"originalLanguage": language.name,
"files": files.map((e) => e.toJson()).toList()
};
}
}

class BatchLocalizationFileData {
final String datName;
final String fileName;
final List<BatchLocalizationEntryData> entries;

BatchLocalizationFileData(this.datName, this.fileName, this.entries);

factory BatchLocalizationFileData.read(StringReader reader) {
final datName = reader.readUntil(_listSeparator);
final fileName = reader.readUntil("\n");
final entries = <BatchLocalizationEntryData>[];
var fileEndIndex = reader.indexOf(_fileSeparator);
while (true) {
var entryEndIndex = reader.indexOf(_entrySeparator);
if (entryEndIndex == -1 || entryEndIndex > fileEndIndex)
break;
entries.add(BatchLocalizationEntryData.read(reader));
}
reader.readUntil(_fileSeparator);
return BatchLocalizationFileData(datName, fileName, entries);
}

factory BatchLocalizationFileData.fromJson(Map json) {
final datName = json["datName"];
final fileName = json["fileName"];
final entries = (json["entries"] as List).map((e) => BatchLocalizationEntryData.fromJson(e)).toList();
return BatchLocalizationFileData(datName, fileName, entries);
}

void writeString(StringSink sink) {
sink.write(datName);
sink.write(_listSeparator);
sink.write(fileName);
sink.writeln();
for (final entry in entries) {
entry.writeString(sink);
}
sink.write(_fileSeparator);
}

Map<String, dynamic> toJson() {
return {
"datName": datName,
"fileName": fileName,
"entries": entries.map((e) => e.toJson()).toList()
};
}

Map<String, String> asMap() {
return {
for (final entry in entries)
entry.key: entry.value
};
}

Map<String, List<String>> asMapOfLists() {
Map<String, List<String>> result = {};
for (final entry in entries) {
if (!result.containsKey(entry.key)) {
result[entry.key] = [];
}
result[entry.key]!.add(entry.value);
}
return result;
}
}

class BatchLocalizationEntryData {
final String key;
final String value;

BatchLocalizationEntryData(this.key, this.value);

factory BatchLocalizationEntryData.read(StringReader reader) {
final key = reader.readUntil(":\n");
final value = reader.readUntil("\n$_entrySeparator");
return BatchLocalizationEntryData(key, value);
}

factory BatchLocalizationEntryData.fromJson(Map json) {
return BatchLocalizationEntryData(json["key"], json["value"]);
}

void writeString(StringSink sink) {
sink.write(key);
sink.write(":\n");
sink.write(value);
sink.writeln();
sink.write(_entrySeparator);
}

Map<String, dynamic> toJson() {
return {
"key": key,
"value": value
};
}
}

const _listSeparator = " -> ";
const _entrySeparator = "----\n";
const _fileSeparator = "====\n";

class StringReader {
final String _string;
int _position = 0;

StringReader(this._string);

bool get isEOF => _position >= _string.length;

int indexOf(String pattern) {
return _string.indexOf(pattern, _position);
}

String readUntil(String pattern) {
final index = _string.indexOf(pattern, _position);
if (index == -1) {
throw Exception("Pattern not found");
}
final result = _string.substring(_position, index);
_position = index + pattern.length;
return result;
}
}
Loading

0 comments on commit 2ba3f06

Please sign in to comment.