oto/apps/runner/test/oto_command_catalog_test.dart
toki 0941d572e5 feat: agent registration capability boundary implementation
- Implement command adapter for G05
- Wire registration components for G06
- Add runner_command_capability_provider
- Update agent_runner and registration_client
- Update tests for registration and command catalog
2026-06-11 09:12:28 +09:00

645 lines
21 KiB
Dart

import 'dart:io';
import 'package:oto/oto/agent/registration_client.dart'
show otoRunnerCapabilityName, otoRunnerCapabilityVersion;
import 'package:oto/oto/commands/command.dart';
import 'package:oto/oto/commands/command_catalog.dart';
import 'package:oto/oto/commands/command_registry.dart';
import 'package:oto/oto/commands/runner_command_capability_provider.dart';
import 'package:test/test.dart';
import 'package:yaml/yaml.dart';
void main() {
setUpAll(() {
registerAllCommands();
});
test('all command types are registered', () {
final registered = Command.registeredTypes.toSet();
final missing = CommandType.values
.where((t) => !registered.contains(t))
.toList();
expect(
missing,
isEmpty,
reason:
'CommandType values without a Command.register() call: $missing. '
'Add registration in the relevant register*() function and call it from registerAllCommands().',
);
});
test('all registered commands expose specs', () {
final specs = Command.specs;
final missing = Command.registeredTypes
.where((t) => !specs.containsKey(t))
.toList();
expect(
missing,
isEmpty,
reason:
'Registered commands without a CommandSpec: $missing. '
'Pass spec: CommandSpec(...) to Command.register().',
);
});
test('spec sample paths exist when provided', () {
final missing = <String>[];
for (final entry in Command.specs.entries) {
final sample = entry.value.samplePath;
if (sample == null) continue;
if (!File(sample).existsSync()) {
missing.add('${entry.key} -> $sample');
}
}
expect(
missing,
isEmpty,
reason:
'CommandSpec.samplePath must point to an existing file. '
'Missing: $missing',
);
});
test('spec sample paths contain the registered command', () {
final mismatches = <String>[];
for (final entry in Command.specs.entries) {
final sample = entry.value.samplePath;
if (sample == null) continue;
final file = File(sample);
if (!file.existsSync()) continue;
final commandName = entry.key.name;
final pattern = RegExp(
r'^\s*-\s*command:\s*' + RegExp.escape(commandName) + r'\s*$',
);
final hasCommand = file.readAsLinesSync().any((line) {
final trimmed = line.trimLeft();
if (trimmed.startsWith('#')) return false;
return pattern.hasMatch(line);
});
if (!hasCommand) {
mismatches.add('${entry.key} -> $sample');
}
}
expect(
mismatches,
isEmpty,
reason:
'CommandSpec.samplePath must reference a sample that actually '
'includes a non-commented `- command: <Type>` line. Mismatches: '
'$mismatches',
);
});
test('spec category and dataModel are non-empty', () {
final bad = <String>[];
for (final entry in Command.specs.entries) {
final spec = entry.value;
if (spec.category.isEmpty || spec.dataModel.isEmpty) {
bad.add('${entry.key}');
}
}
expect(
bad,
isEmpty,
reason: 'CommandSpec.category and dataModel must be non-empty: $bad',
);
});
// REVIEW_SAMPLE-3: stale iOS field name regression tests
// These tests guard against stale/old field names that no longer match
// the actual data models (DataBuildiOS, DataArchiveiOS, DataExportiOS,
// DataTestflightUpload, DataTestflightStatusCheck, DataTestflightDistribute).
// Stale field names that should NOT appear in any sample YAML:
// - build_ios.yaml: `workspacePath` was replaced with `xcworkspaceFilePath`/`xcodeProjectFilePath`
// - build_ios.yaml: `exportOptionsPlist` was replaced with `exportOptionPlistPath`
// - build_ios.yaml: `username`/`password` were replaced with `apiKeyPath`/`appID` for TestFlight
test('iOS sample must not contain stale field: workspacePath', () {
const samplePath = 'assets/yaml/sample/10_build_ios.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
bool hasStaleWorkspacePath = false;
for (final line in content.split('\n')) {
if (line.trimLeft().startsWith('#')) continue;
if (RegExp(r'^[ \t]+workspacePath\s*:').hasMatch(line)) {
hasStaleWorkspacePath = true;
break;
}
}
expect(
hasStaleWorkspacePath,
isFalse,
reason:
'10_build_ios.yaml contains stale field name `workspacePath`. '
'Use `xcworkspaceFilePath` or `xcodeProjectFilePath` to match DataBuildiOS/DataArchiveiOS.',
);
});
test('iOS sample must not contain stale field: exportOptionsPlist', () {
const samplePath = 'assets/yaml/sample/10_build_ios.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
final lines = content.split('\n');
bool hasStaleExportOptionsPlist = false;
for (final line in lines) {
if (line.trimLeft().startsWith('#')) continue;
if (RegExp(r'^[ \t]+exportOptionsPlist\s*:').hasMatch(line)) {
hasStaleExportOptionsPlist = true;
break;
}
}
expect(
hasStaleExportOptionsPlist,
isFalse,
reason:
'10_build_ios.yaml contains stale field name `exportOptionsPlist`. '
'Use `exportOptionPlistPath` to match DataExportiOS.',
);
});
test('iOS TestFlight samples must not contain stale field: username/password', () {
const samplePath = 'assets/yaml/sample/10_build_ios.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
final lines = content.split('\n');
// Check for stale `username:` or `password:` YAML keys (comment lines excluded)
bool hasStaleUser = false;
bool hasStalePass = false;
for (final line in lines) {
if (line.trimLeft().startsWith('#')) continue;
if (RegExp(r'^[ \t]+username\s*:').hasMatch(line)) {
hasStaleUser = true;
break;
}
if (RegExp(r'^[ \t]+password\s*:').hasMatch(line)) {
hasStalePass = true;
break;
}
}
expect(
hasStaleUser || hasStalePass,
isFalse,
reason:
'10_build_ios.yaml contains stale TestFlight field names `username`/`password`. '
'DataTestflightUpload expects `apiKeyPath` and `ipaPath`. '
'DataTestflightStatusCheck expects `appID`, `apiKeyPath`, `version`, `buildNumber`. '
'DataTestflightDistribute expects `apiKeyPath`, `appID`. '
'Use `apiKeyPath` instead of `username`/`password`.',
);
});
test('sample fields must include expected iOS model fields', () {
const samplePath = 'assets/yaml/sample/10_build_ios.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
// DataBuildiOS field names that should be present:
// scheme, configuration, xcworkspaceFilePath, xcodeProjectFilePath
// DataArchiveiOS field names that should be present:
// xcworkspaceFilePath, xcodeProjectFilePath, scheme, archivePath
// DataExportiOS field names that should be present:
// archivePath, exportOptionPlistPath, exportPath
final expectedFields = [
'xcworkspaceFilePath', // DataBuildiOS, DataArchiveiOS
'scheme', // DataBuildiOS, DataArchiveiOS
'configuration', // DataBuildiOS, DataArchiveiOS
'archivePath', // DataArchiveiOS, DataExportiOS
'exportOptionPlistPath', // DataExportiOS
'exportPath', // DataExportiOS
'apiKeyPath', // DataTestflightUpload, DataTestflightStatusCheck, DataTestflightDistribute
'appID', // DataTestflightStatusCheck, DataTestflightDistribute
];
final missingFields = <String>[];
for (final field in expectedFields) {
// Look for field as a YAML key: optional whitespace, then the field name
// followed by colon, not inside a comment
final pattern = RegExp(r'^[ \t]+' + RegExp.escape(field) + r'\s*:');
final lines = content.split('\n');
bool found = false;
for (final line in lines) {
if (line.trimLeft().startsWith('#')) continue;
if (pattern.hasMatch(line)) {
found = true;
break;
}
}
if (!found) {
missingFields.add(field);
}
}
expect(
missingFields,
isEmpty,
reason:
'10_build_ios.yaml should contain fields matching actual data models. '
'Missing fields: $missingFields',
);
});
test('all registered commands have samplePath (with allowlist)', () {
// These commands intentionally have no sample yet:
// - SimpleCommand: Internal utility base class, not user-facing
// - CreateAppData: Internal app data generation
// - BuildDartCompile: Internal build helper
// - SMBAuth: Windows-only infra command
final allowlistedWithoutSample = {
'SimpleCommand',
'CreateAppData',
'BuildDartCompile',
'SMBAuth',
};
final missingSample = <String>[];
for (final entry in Command.specs.entries) {
final sample = entry.value.samplePath;
final typeName = entry.key.name;
if (sample == null) {
if (!allowlistedWithoutSample.contains(typeName)) {
missingSample.add('${entry.key} (no samplePath, not in allowlist)');
}
}
}
expect(
missingSample,
isEmpty,
reason:
'Registered public commands without samplePath must be explicitly allowlisted. '
'Missing: $missingSample. '
'If a command was intentionally not allowlisted, add a sample file and '
'specify samplePath in the CommandSpec.',
);
});
test(
'all non-allowlisted commands have sample files containing the registered command',
() {
final allowlistedWithoutSample = {
'SimpleCommand',
'CreateAppData',
'BuildDartCompile',
'SMBAuth',
};
final missingContent = <String>[];
for (final entry in Command.specs.entries) {
final sample = entry.value.samplePath;
if (sample == null) continue;
final typeName = entry.key.name;
if (allowlistedWithoutSample.contains(typeName)) continue;
final file = File(sample);
if (!file.existsSync()) continue;
final pattern = RegExp(
r'^\s*-\s*command:\s*' + RegExp.escape(entry.key.name) + r'\s*$',
);
final hasCommand = file.readAsLinesSync().any((line) {
final trimmed = line.trimLeft();
if (trimmed.startsWith('#')) return false;
return pattern.hasMatch(line);
});
if (!hasCommand) {
missingContent.add('${entry.key} -> $sample');
}
}
expect(
missingContent,
isEmpty,
reason:
'All non-allowlisted samplePath files must contain at least one non-commented `- command: <Type>` line. '
'Missing: $missingContent',
);
},
);
test(
'GitHub PR samples use write tags (<@>) not read tags (<!>) for result storage',
() {
const samplePath = 'assets/yaml/sample/09_network.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
// Locate GitHub PR command blocks and check their write fields
final lines = content.split('\n');
bool inGithubCreate = false;
bool inGithubList = false;
for (final line in lines) {
// Track whether we're inside a GitHub PR command block
if (line.contains(r'- command: GitHubPullRequestCreate') ||
line.contains('- command: GitHubPullRequestList')) {
inGithubCreate = line.contains('GitHubPullRequestCreate');
inGithubList = !inGithubCreate;
} else if (line.contains('- command: ') &&
!line.contains('GitHubPullRequest')) {
inGithubCreate = false;
inGithubList = false;
}
// In PR create/list blocks, setURL/setNumber/setValue must use write tag <@>, not read tag <!>
if (inGithubCreate || inGithubList) {
final trimmed = line.trimLeft();
if (trimmed.startsWith('setURL:') ||
trimmed.startsWith('setNumber:') ||
trimmed.startsWith('setValue:')) {
// Read tag <!property... is wrong here — must be write tag <@property...>
if (RegExp(r'<![^>]+>').hasMatch(trimmed)) {
fail(
'GitHub PR command write fields must use write tags '
'(<@property...>) not read tags (<!property...>). '
'Found: $trimmed',
);
}
}
}
}
},
);
test('all samplePath YAML files parse as maps with a commands list', () {
final parseFailures = <String>[];
for (final entry in Command.specs.entries) {
final sample = entry.value.samplePath;
if (sample == null) continue;
final file = File(sample);
if (!file.existsSync()) continue;
try {
final content = file.readAsStringSync();
final node = loadYaml(content);
if (node == null || node is! Map) {
parseFailures.add('${entry.key} -> $sample: root is not a YAML map');
continue;
}
if (!node.containsKey('commands') || node['commands'] == null) {
parseFailures.add('${entry.key} -> $sample: missing commands list');
continue;
}
final commands = node['commands'];
if (commands is! List) {
parseFailures.add('${entry.key} -> $sample: commands is not a list');
continue;
}
for (var i = 0; i < commands.length; i++) {
final cmd = commands[i];
if (cmd is! Map || !cmd.containsKey('command')) {
parseFailures.add(
'${entry.key} -> $sample: commands[$i] missing command key',
);
break;
}
}
} catch (e) {
parseFailures.add('${entry.key} -> $sample: YAML parse error: $e');
}
}
expect(
parseFailures,
isEmpty,
reason:
'Sample YAML files must parse as valid maps with commands lists. Failures: $parseFailures',
);
});
test('README examples must not contain stale field workspacePath', () {
final file = File('README.md');
if (!file.existsSync()) {
fail('README.md not found');
}
final content = file.readAsStringSync();
for (final line in content.split('\n')) {
if (line.trimLeft().startsWith('#')) continue;
expect(
RegExp(r'^[ \t]+workspacePath\s*:').hasMatch(line),
isFalse,
reason:
'README.md contains stale field `workspacePath`. '
'Use `xcworkspaceFilePath` or `xcodeProjectFilePath` to match DataBuildiOS/DataArchiveiOS.',
);
}
});
test('AwsCli sample uses sub-command key (not subCommand)', () {
const samplePath = 'assets/yaml/sample/19_awscli.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
final node = loadYaml(content) as Map;
final commands = node['commands'] as List;
final awsCliBlock = commands.firstWhere(
(c) => (c as Map)['command'] == 'AwsCli',
orElse: () => null,
);
expect(
awsCliBlock,
isNotNull,
reason: '19_awscli.yaml must contain an AwsCli command block',
);
final param = awsCliBlock['param'] as Map;
expect(
param.containsKey('sub-command'),
isTrue,
reason:
'AwsCli sample must use the key sub-command (not subCommand) to match DataAwsCli @JsonKey(name: "sub-command")',
);
expect(
param.containsKey('subCommand'),
isFalse,
reason:
'AwsCli sample must not use the key subCommand; use sub-command instead',
);
});
test('Jira sample includes both id and token fields', () {
const samplePath = 'assets/yaml/sample/21_jira.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
final node = loadYaml(content) as Map;
final commands = node['commands'] as List;
final jiraBlock = commands.firstWhere(
(c) => (c as Map)['command'] == 'Jira',
orElse: () => null,
);
expect(
jiraBlock,
isNotNull,
reason: '21_jira.yaml must contain a Jira command block',
);
final param = jiraBlock['param'] as Map;
expect(
param.containsKey('id'),
isTrue,
reason: 'Jira sample must include the id field for Jira domain/account',
);
expect(
param.containsKey('token'),
isTrue,
reason:
'Jira sample must include the token field for Jira API authentication',
);
});
test('Protobuf sample commands list contains valid string entries', () {
const samplePath = 'assets/yaml/sample/18_protobuf.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
final node = loadYaml(content) as Map;
final commands = node['commands'] as List;
final protoBlock = commands.firstWhere(
(c) => (c as Map)['command'] == 'Protobuf',
orElse: () => null,
);
expect(
protoBlock,
isNotNull,
reason: '18_protobuf.yaml must contain a Protobuf command block',
);
final param = protoBlock['param'] as Map;
expect(
param.containsKey('commands'),
isTrue,
reason: 'Protobuf sample must include a commands field',
);
final cmdList = param['commands'] as List;
expect(
cmdList.isNotEmpty,
isTrue,
reason: 'Protobuf commands list must not be empty',
);
for (var item in cmdList) {
expect(
item is String,
isTrue,
reason:
'Protobuf commands list must contain string entries. Got: $item (${item.runtimeType})',
);
}
});
test('BuildFlutter sample uses supported fields and flutter targets', () {
const samplePath = 'assets/yaml/sample/11_build_flutter.yaml';
final file = File(samplePath);
if (!file.existsSync()) {
fail('Sample file not found: $samplePath');
}
final content = file.readAsStringSync();
final node = loadYaml(content) as Map;
final commands = node['commands'] as List;
final buildFlutterBlocks = commands
.where((c) => (c as Map)['command'] == 'BuildFlutter')
.toList();
expect(
buildFlutterBlocks.isNotEmpty,
isTrue,
reason: '11_build_flutter.yaml must contain BuildFlutter command blocks',
);
final allowedPlatforms = {
'ios',
'apk',
'appbundle',
'web',
'macos',
'windows',
'linux',
};
for (var block in buildFlutterBlocks) {
final param = (block as Map)['param'] as Map;
expect(
param.containsKey('targetPath'),
isFalse,
reason:
'BuildFlutter sample must not use targetPath (stale field). Use workspace instead.',
);
expect(
param.containsKey('workspace'),
isTrue,
reason: 'BuildFlutter sample must define workspace.',
);
expect(
param.containsKey('platform'),
isTrue,
reason: 'BuildFlutter sample must define platform.',
);
final platform = param['platform']?.toString().toLowerCase();
expect(
allowedPlatforms.contains(platform),
isTrue,
reason:
'BuildFlutter platform must be one of $allowedPlatforms. Got: $platform',
);
}
});
test('command catalog contract exposes stable fields', () {
final entries = const CommandCatalog().entries();
expect(entries, isNotEmpty);
final firstJson = entries.first.toJson();
expect(
firstJson.keys,
containsAll([
'command',
'category',
'dataModel',
'samplePath',
'hasSample',
'sampleExists',
]),
);
expect(firstJson['command'], isA<String>());
expect(firstJson['category'], isA<String>());
expect(firstJson['dataModel'], isA<String>());
expect(firstJson['hasSample'], isA<bool>());
expect(firstJson['sampleExists'], isA<bool>());
});
test('command capability provider mirrors command catalog types', () {
final snapshot = const CommandCatalogRunnerCapabilityProvider().snapshot();
final catalogTypes = Command.catalogRows
.map((row) => row['type']!)
.toList();
expect(snapshot.name, otoRunnerCapabilityName);
expect(snapshot.version, otoRunnerCapabilityVersion);
expect(snapshot.commandTypes, catalogTypes);
});
test('command catalog filters category and command case-insensitively', () {
final catalog = const CommandCatalog();
final buildEntries = catalog.entries(category: 'BUILD');
expect(buildEntries, isNotEmpty);
expect(buildEntries.every((entry) => entry.category == 'build'), isTrue);
final printEntries = catalog.entries(command: 'print');
expect(printEntries.map((entry) => entry.command), ['Print']);
});
}