Skip to content

Commit

Permalink
updated cleanCode to assume code inside markdown dart code block
Browse files Browse the repository at this point in the history
  • Loading branch information
csells committed Feb 18, 2025
1 parent 27a8f82 commit ae66154
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 87 deletions.
126 changes: 66 additions & 60 deletions pkgs/dart_services/lib/src/generative_ai.dart
Original file line number Diff line number Diff line change
Expand Up @@ -203,77 +203,80 @@ $prompt
.where((text) => text.isNotEmpty);

static const startCodeBlock = '```dart\n';
static const endCodeBlock = '```';

static Stream<String> cleanCode(Stream<String> stream) async* {
var foundFirstLine = false;
static const endCodeBlock = '\n```';

/// Parses a stream of markdown text and yields only the content inside a
/// ```dart ... ``` code block.
///
/// Any text before the first occurrence of "```dart" is ignored. Once inside
/// the code block, text is yielded until the closing "```" is encountered,
/// at which point any remaining text is ignored.
///
/// This parser works in a streaming manner and does not assume that the start
/// or end markers are contained entirely in one chunk.
static Stream<String> cleanCode(Stream<String> input) async* {
const startMarker = '```dart\n';
const endMarker = '```';
final buffer = StringBuffer();
await for (final chunk in stream) {
// looking for the start of the code block (if there is one)
if (!foundFirstLine) {
buffer.write(chunk);
if (chunk.contains('\n')) {
foundFirstLine = true;
final text = buffer.toString().replaceFirst(startCodeBlock, '');
buffer.clear();
if (text.isNotEmpty) yield text;
continue;
}
var foundStart = false;
var foundEnd = false;

// still looking for the start of the first line
continue;
}
await for (final chunk in input) {
if (foundEnd) continue;
buffer.write(chunk);

if (!foundStart) {
final str = buffer.toString();
final startIndex = str.indexOf(startMarker);
if (startIndex == -1) continue;

// looking for the end of the code block (if there is one)
assert(foundFirstLine);
String processedChunk;
if (chunk.endsWith(endCodeBlock)) {
processedChunk = chunk.substring(0, chunk.length - endCodeBlock.length);
} else if (chunk.endsWith('$endCodeBlock\n')) {
processedChunk =
'${chunk.substring(0, chunk.length - endCodeBlock.length - 1)}\n';
} else {
processedChunk = chunk;
buffer.clear();
buffer.write(str.substring(startIndex + startMarker.length));
foundStart = true;
}

if (processedChunk.isNotEmpty) yield processedChunk;
}
assert(foundStart);
assert(!foundEnd);

// if we're still in the first line, yield it
if (buffer.isNotEmpty) yield buffer.toString();
final str = buffer.toString();
final endIndex = str.indexOf(endMarker);
foundEnd = endIndex != -1;
final output = foundEnd ? str.substring(0, endIndex) : str;
yield output;
buffer.clear();
}
}
}

final _cachedAllowedFlutterPackages = List<String>.empty(growable: true);
List<String> _allowedFlutterPackages() {
if (_cachedAllowedFlutterPackages.isEmpty) {
final versions = getPackageVersions();
for (final MapEntry(key: name, value: version) in versions.entries) {
if (isSupportedPackage(name)) {
_cachedAllowedFlutterPackages.add('$name: $version');
final _cachedAllowedFlutterPackages = List<String>.empty(growable: true);
List<String> _allowedFlutterPackages() {
if (_cachedAllowedFlutterPackages.isEmpty) {
final versions = getPackageVersions();
for (final MapEntry(key: name, value: version) in versions.entries) {
if (isSupportedPackage(name)) {
_cachedAllowedFlutterPackages.add('$name: $version');
}
}
}
}

return _cachedAllowedFlutterPackages;
}
return _cachedAllowedFlutterPackages;
}

final _cachedAllowedDartPackages = List<String>.empty(growable: true);
List<String> _allowedDartPackages() {
if (_cachedAllowedDartPackages.isEmpty) {
final versions = getPackageVersions();
for (final MapEntry(key: name, value: version) in versions.entries) {
if (isSupportedDartPackage(name)) {
_cachedAllowedDartPackages.add('$name: $version');
final _cachedAllowedDartPackages = List<String>.empty(growable: true);
List<String> _allowedDartPackages() {
if (_cachedAllowedDartPackages.isEmpty) {
final versions = getPackageVersions();
for (final MapEntry(key: name, value: version) in versions.entries) {
if (isSupportedDartPackage(name)) {
_cachedAllowedDartPackages.add('$name: $version');
}
}
}
}

return _cachedAllowedDartPackages;
}
return _cachedAllowedDartPackages;
}

Content _flutterSystemInstructions(String modelSpecificInstructions) =>
Content.text('''
Content _flutterSystemInstructions(String modelSpecificInstructions) =>
Content.text('''
You're an expert Flutter developer and UI designer creating Custom User
Interfaces: generated, bespoke, interactive interfaces created on-the-fly using
the Flutter SDK API. You will produce a professional, release-ready Flutter
Expand Down Expand Up @@ -443,11 +446,12 @@ ${_allowedFlutterPackages().map((p) => '- $p').join('\n')}
$modelSpecificInstructions
Only output the Dart code for the program.
Only output the Dart code for the program. Output that code wrapped in a
Markdown ```dart``` tag.
''');

Content _dartSystemInstructions(String modelSpecificInstructions) =>
Content.text('''
Content _dartSystemInstructions(String modelSpecificInstructions) =>
Content.text('''
You're an expert Dart developer specializing in writing efficient, idiomatic,
and production-ready Dart programs.
You will produce professional, release-ready Dart applications. All of the
Expand Down Expand Up @@ -494,7 +498,7 @@ and provide meaningful default values when needed.
- **Ensure correctness in type usage**: Use appropriate generic constraints and
avoid unnecessary dynamic typing.
- If the program requires **parsing, async tasks, or JSON handling**, use Darts
- If the program requires **parsing, async tasks, or JSON handling**, use Dart's
built-in libraries like `dart:convert` and `dart:async` instead of external
dependencies unless specified.
Expand Down Expand Up @@ -544,5 +548,7 @@ $modelSpecificInstructions
---
Only output the Dart code for the program.
Only output the Dart code for the program. Output that code wrapped in a
Markdown ```dart``` tag.
''');
}
53 changes: 31 additions & 22 deletions pkgs/dart_services/test/genai_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -7,65 +7,74 @@ import 'package:test/test.dart';

void main() {
group('GenerativeAI.cleanCode', () {
final unwrappedCode = '''
final code = '''
void main() {
print("hello, world");
}
''';
final wrappedCode = '''
```dart
$code
```
''';

test('handles code without markdown wrapper', () async {
test('handles code with markdown wrapper', () async {
final input = Stream.fromIterable(
unwrappedCode.split('\n').map((line) => '$line\n'),
wrappedCode.split('\n').map((line) => '$line\n'),
);
final cleaned = await GenerativeAI.cleanCode(input).join();
expect(cleaned.trim(), unwrappedCode.trim());
expect(cleaned.trim(), code.trim());
});

test('handles code with markdown wrapper', () async {
test('handles code with markdown wrapper and some leading gunk', () async {
final input = Stream.fromIterable(
unwrappedCode.split('\n').map((line) => '$line\n'),
[
'some leading gunk\n',
...wrappedCode.split('\n').map((line) => '$line\n')
],
);
final cleaned = await GenerativeAI.cleanCode(input).join();
expect(cleaned.trim(), unwrappedCode.trim());
expect(cleaned.trim(), code.trim());
});

test('handles code with markdown wrapper and trailing newline', () async {
test('handles code with markdown wrapper and trailing gunk', () async {
final input = Stream.fromIterable(
unwrappedCode.split('\n').map((line) => '$line\n'),
[
...wrappedCode.split('\n').map((line) => '$line\n'),
'some trailing gunk\n',
],
);
final cleaned = await GenerativeAI.cleanCode(input).join();
expect(cleaned.trim(), unwrappedCode.trim());
expect(cleaned.trim(), code.trim());
});

test('handles single-chunk response with markdown wrapper', () async {
final input = Stream.fromIterable(
unwrappedCode.split('\n').map((line) => '$line\n'),
);
final input = Stream.fromIterable([wrappedCode]);
final cleaned = await GenerativeAI.cleanCode(input).join();
expect(cleaned.trim(), unwrappedCode.trim());
expect(cleaned.trim(), code.trim());
});

test('handles partial first line buffering', () async {
final input = Stream.fromIterable([
'```',
'dart\n',
'void main() {\n',
' print("hello, world");\n',
'}\n',
...code.split('\n').map((line) => '$line\n'),
'```',
]);

final cleaned = await GenerativeAI.cleanCode(input).join();
expect(cleaned.trim(), unwrappedCode.trim());
expect(cleaned.trim(), code.trim());
});

test('handles single-line code without trailing newline', () async {
final input = Stream.fromIterable(
['void main() { print("hello, world"); }'],
);
final input = Stream.fromIterable([
'```dart\n',
'void main() { print("hello, world"); }',
'```',
]);

final cleaned = await GenerativeAI.cleanCode(input).join();
final oneline = unwrappedCode
final oneline = code
.replaceAll('\n', ' ')
.replaceAll(' ', ' ')
.replaceAll(' ', ' ');
Expand Down
3 changes: 1 addition & 2 deletions pkgs/dartpad_ui/lib/enable_gen_ai.dart
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
// turn on or off gen-ai features in the client
// TODO(csells): SET TO FALSE WHEN DONE TESTING
const bool genAiEnabled = true;
const bool genAiEnabled = false;
4 changes: 1 addition & 3 deletions pkgs/dartpad_ui/lib/model.dart
Original file line number Diff line number Diff line change
Expand Up @@ -571,9 +571,7 @@ enum Channel {

const Channel(this.displayName, this.url);

// TODO(csells): REMOVE
// static const defaultChannel = Channel.stable;
static const defaultChannel = Channel.localhost;
static const defaultChannel = Channel.stable;

static List<Channel> get valuesWithoutLocalhost {
return values.whereNot((channel) => channel == localhost).toList();
Expand Down
3 changes: 3 additions & 0 deletions pkgs/dartpad_ui/lib/widgets.dart
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,7 @@ class GeneratingCodeDialog extends StatefulWidget {

class _GeneratingCodeDialogState extends State<GeneratingCodeDialog> {
final _generatedCode = StringBuffer();
final _focusNode = FocusNode();
bool _done = false;
StreamSubscription<String>? _subscription;

Expand All @@ -537,6 +538,7 @@ class _GeneratingCodeDialogState extends State<GeneratingCodeDialog> {
_generatedCode.clear();
_generatedCode.write(source);
_done = true;
_focusNode.requestFocus();
}),
);
}
Expand Down Expand Up @@ -581,6 +583,7 @@ class _GeneratingCodeDialogState extends State<GeneratingCodeDialog> {
width: 700,
child: Focus(
autofocus: true,
focusNode: _focusNode,
child: widget.existingSource == null
? ReadOnlyEditorWidget(_generatedCode.toString())
: ReadOnlyDiffWidget(
Expand Down

0 comments on commit ae66154

Please sign in to comment.