diff --git a/completions-sample-code/common/ghostTextContext.ts b/completions-sample-code/common/ghostTextContext.ts new file mode 100644 index 0000000..2563939 --- /dev/null +++ b/completions-sample-code/common/ghostTextContext.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { InlineEditRequestLogContext } from '../../../platform/inlineEdits/common/inlineEditLogContext'; +import { basename } from '../../../util/vs/base/common/path'; + +export class GhostTextContext extends InlineEditRequestLogContext { + override getDebugName(): string { + return `Ghost | ${basename(this.filePath)} (v${this.version})`; + } +} diff --git a/completions-sample-code/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts b/completions-sample-code/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts new file mode 100644 index 0000000..67f0d87 --- /dev/null +++ b/completions-sample-code/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts @@ -0,0 +1,76 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ITelemetryService, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../../../platform/telemetry/common/telemetry'; +import { wrapEventNameForPrefixRemoval } from '../../../../../platform/telemetry/node/azureInsightsReporter'; +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { TelemetryMeasurements, TelemetryProperties, TelemetryStore } from '../../lib/src/telemetry'; +import type { TelemetrySpy } from '../../lib/src/test/telemetrySpy'; + +export const ICompletionsTelemetryService = createServiceIdentifier('completionsTelemetryService'); +export interface ICompletionsTelemetryService { + readonly _serviceBrand: undefined; + + sendGHTelemetryEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void; + sendEnhancedGHTelemetryEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void; + sendGHTelemetryErrorEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void; + sendGHTelemetryException(maybeError: unknown, origin: string, store?: TelemetryStore): void; + setSpyReporters(reporter: TelemetrySpy, enhancedReporter: TelemetrySpy): void; + clearSpyReporters(): void; +} + +export class CompletionsTelemetryServiceBridge implements ICompletionsTelemetryService { + declare _serviceBrand: undefined; + + private reporter: TelemetrySpy | undefined; + private enhancedReporter: TelemetrySpy | undefined; + + constructor( + @ITelemetryService private readonly telemetryService: ITelemetryService + ) { + this.reporter = undefined; + this.enhancedReporter = undefined; + } + + sendGHTelemetryEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void { + this.telemetryService.sendGHTelemetryEvent(wrapEventNameForPrefixRemoval(`copilot/${eventName}`), properties, measurements); + this.getSpyReporters(store ?? TelemetryStore.Standard)?.sendTelemetryEvent(eventName, properties as TelemetryProperties, measurements as TelemetryMeasurements); + } + + sendEnhancedGHTelemetryEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void { + this.telemetryService.sendEnhancedGHTelemetryEvent(wrapEventNameForPrefixRemoval(`copilot/${eventName}`), properties, measurements); + this.getSpyReporters(store ?? TelemetryStore.Enhanced)?.sendTelemetryEvent(eventName, properties as TelemetryProperties, measurements as TelemetryMeasurements); + } + + sendGHTelemetryErrorEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void { + this.telemetryService.sendGHTelemetryErrorEvent(wrapEventNameForPrefixRemoval(`copilot/${eventName}`), properties, measurements); + this.getSpyReporters(store ?? TelemetryStore.Enhanced)?.sendTelemetryErrorEvent(eventName, properties as TelemetryProperties, measurements as TelemetryMeasurements); + } + + sendGHTelemetryException(maybeError: unknown, origin: string, store?: TelemetryStore): void { + this.telemetryService.sendGHTelemetryException(maybeError, origin); + if (maybeError instanceof Error) { + this.getSpyReporters(store ?? TelemetryStore.Enhanced)?.sendTelemetryException(maybeError as Error, undefined, undefined); + } + } + + setSpyReporters(reporter: TelemetrySpy, enhancedReporter: TelemetrySpy) { + this.reporter = reporter; + this.enhancedReporter = enhancedReporter; + } + + clearSpyReporters() { + this.reporter = undefined; + this.enhancedReporter = undefined; + } + + private getSpyReporters(store: TelemetryStore): TelemetrySpy | undefined { + if (TelemetryStore.isEnhanced(store)) { + return this.enhancedReporter; + } else { + return this.reporter; + } + } +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/completionsServiceBridges.ts b/completions-sample-code/vscode-node/completionsServiceBridges.ts new file mode 100644 index 0000000..d9d896a --- /dev/null +++ b/completions-sample-code/vscode-node/completionsServiceBridges.ts @@ -0,0 +1,262 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { commands, env } from 'vscode'; +import { ILogService } from '../../../platform/log/common/logService'; +import { outputChannel } from '../../../platform/log/vscode/outputChannelLogTarget'; +import { DisposableStore, IDisposable } from '../../../util/vs/base/common/lifecycle'; +import { URI } from '../../../util/vs/base/common/uri'; +import { SyncDescriptor } from '../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService, ServicesAccessor } from '../../../util/vs/platform/instantiation/common/instantiation'; +import { ServiceCollection } from '../../../util/vs/platform/instantiation/common/serviceCollection'; +import { CompletionsTelemetryServiceBridge, ICompletionsTelemetryService } from './bridge/src/completionsTelemetryServiceBridge'; +import { LoggingCitationManager } from './extension/src/codeReferencing/citationManager'; +import { CompletionsObservableWorkspace } from './extension/src/completionsObservableWorkspace'; +import { disableCompletions, enableCompletions, toggleCompletions, VSCodeConfigProvider, VSCodeEditorInfo } from './extension/src/config'; +import { CMDDisableCompletionsChat, CMDDisableCompletionsClient, CMDEnableCompletionsChat, CMDEnableCompletionsClient, CMDOpenDocumentationClient, CMDOpenLogsClient, CMDOpenModelPickerChat, CMDOpenModelPickerClient, CMDToggleCompletionsChat, CMDToggleCompletionsClient, CMDToggleStatusMenuChat, CMDToggleStatusMenuClient } from './extension/src/constants'; +import { contextProviderMatch } from './extension/src/contextProviderMatch'; +import { registerPanelSupport } from './extension/src/copilotPanel/common'; +import { CopilotExtensionStatus, ICompletionsExtensionStatus } from './extension/src/extensionStatus'; +import { extensionFileSystem } from './extension/src/fileSystem'; +import { ModelPickerManager } from './extension/src/modelPicker'; +import { CopilotStatusBar } from './extension/src/statusBar'; +import { CopilotStatusBarPickMenu } from './extension/src/statusBarPicker'; +import { ExtensionTextDocumentManager } from './extension/src/textDocumentManager'; +import { exception } from './extension/src/vscodeInlineCompletionItemProvider'; +import { CopilotTokenManagerImpl, ICompletionsCopilotTokenManager } from './lib/src/auth/copilotTokenManager'; +import { ICompletionsCitationManager } from './lib/src/citationManager'; +import { CompletionNotifier, ICompletionsNotifierService } from './lib/src/completionNotifier'; +import { ICompletionsObservableWorkspace } from './lib/src/completionsObservableWorkspace'; +import { ICompletionsConfigProvider, ICompletionsEditorAndPluginInfo } from './lib/src/config'; +import { registerDocumentTracker } from './lib/src/documentTracker'; +import { ICompletionsUserErrorNotifierService, UserErrorNotifier } from './lib/src/error/userErrorNotifier'; +import { setupCompletionsExperimentationService } from './lib/src/experiments/defaultExpFilters'; +import { Features } from './lib/src/experiments/features'; +import { ICompletionsFeaturesService } from './lib/src/experiments/featuresService'; +import { FileReader, ICompletionsFileReaderService } from './lib/src/fileReader'; +import { ICompletionsFileSystemService } from './lib/src/fileSystem'; +import { AsyncCompletionManager, ICompletionsAsyncManagerService } from './lib/src/ghostText/asyncCompletions'; +import { CompletionsCache, ICompletionsCacheService } from './lib/src/ghostText/completionsCache'; +import { ConfigBlockModeConfig, ICompletionsBlockModeConfig } from './lib/src/ghostText/configBlockMode'; +import { CurrentGhostText, ICompletionsCurrentGhostText } from './lib/src/ghostText/current'; +import { ICompletionsLastGhostText, LastGhostText } from './lib/src/ghostText/last'; +import { ICompletionsSpeculativeRequestCache, SpeculativeRequestCache } from './lib/src/ghostText/speculativeRequestCache'; +import { ICompletionsLogTargetService, LogLevel } from './lib/src/logger'; +import { formatLogMessage } from './lib/src/logging/util'; +import { CompletionsFetcher, ICompletionsFetcherService } from './lib/src/networking'; +import { ExtensionNotificationSender, ICompletionsNotificationSender } from './lib/src/notificationSender'; +import { ICompletionsOpenAIFetcherService, LiveOpenAIFetcher } from './lib/src/openai/fetch'; +import { AvailableModelsManager, ICompletionsModelManagerService } from './lib/src/openai/model'; +import { ICompletionsStatusReporter } from './lib/src/progress'; +import { + CompletionsPromptFactory, ICompletionsPromptFactoryService +} from './lib/src/prompt/completionsPromptFactory/completionsPromptFactory'; +import { ContextProviderBridge, ICompletionsContextProviderBridgeService } from './lib/src/prompt/components/contextProviderBridge'; +import { + CachedContextProviderRegistry, + CoreContextProviderRegistry, + DefaultContextProvidersContainer, ICompletionsContextProviderRegistryService, + ICompletionsDefaultContextProviders +} from './lib/src/prompt/contextProviderRegistry'; +import { ContextProviderStatistics, ICompletionsContextProviderService } from './lib/src/prompt/contextProviderStatistics'; +import { FullRecentEditsProvider, ICompletionsRecentEditsProviderService } from './lib/src/prompt/recentEdits/recentEditsProvider'; +import { CompositeRelatedFilesProvider } from './lib/src/prompt/similarFiles/compositeRelatedFilesProvider'; +import { ICompletionsRelatedFilesProviderService } from './lib/src/prompt/similarFiles/relatedFiles'; +import { ICompletionsTelemetryUserConfigService, TelemetryUserConfig } from './lib/src/telemetry/userConfig'; +import { ICompletionsTextDocumentManagerService } from './lib/src/textDocumentManager'; +import { ICompletionsPromiseQueueService, PromiseQueue } from './lib/src/util/promiseQueue'; +import { ICompletionsRuntimeModeService, RuntimeMode } from './lib/src/util/runtimeMode'; + +/** @public */ +export function createContext(serviceAccessor: ServicesAccessor, store: DisposableStore): IInstantiationService { + const logService = serviceAccessor.get(ILogService); + + const serviceCollection = new ServiceCollection(); + + serviceCollection.set(ICompletionsLogTargetService, new class implements ICompletionsLogTargetService { + declare _serviceBrand: undefined; + logIt(level: LogLevel, category: string, ...extra: unknown[]): void { + const msg = formatLogMessage(category, ...extra); + switch (level) { + case LogLevel.DEBUG: return logService.debug(msg); + case LogLevel.INFO: return logService.info(msg); + case LogLevel.WARN: return logService.warn(msg); + case LogLevel.ERROR: return logService.error(msg); + } + } + }); + + serviceCollection.set(ICompletionsRuntimeModeService, RuntimeMode.fromEnvironment(false)); + serviceCollection.set(ICompletionsCacheService, new CompletionsCache()); + serviceCollection.set(ICompletionsConfigProvider, new VSCodeConfigProvider()); + serviceCollection.set(ICompletionsLastGhostText, new LastGhostText()); + serviceCollection.set(ICompletionsCurrentGhostText, new CurrentGhostText()); + serviceCollection.set(ICompletionsSpeculativeRequestCache, new SpeculativeRequestCache()); + serviceCollection.set(ICompletionsNotificationSender, new SyncDescriptor(ExtensionNotificationSender)); + serviceCollection.set(ICompletionsEditorAndPluginInfo, new VSCodeEditorInfo()); + serviceCollection.set(ICompletionsExtensionStatus, new CopilotExtensionStatus()); + serviceCollection.set(ICompletionsFeaturesService, new SyncDescriptor(Features)); + serviceCollection.set(ICompletionsObservableWorkspace, new SyncDescriptor(CompletionsObservableWorkspace)); + serviceCollection.set(ICompletionsStatusReporter, new SyncDescriptor(CopilotStatusBar, ['github.copilot.languageStatus'])); + serviceCollection.set(ICompletionsCopilotTokenManager, new SyncDescriptor(CopilotTokenManagerImpl, [false])); + serviceCollection.set(ICompletionsTextDocumentManagerService, new SyncDescriptor(ExtensionTextDocumentManager)); + serviceCollection.set(ICompletionsFileReaderService, new SyncDescriptor(FileReader)); + serviceCollection.set(ICompletionsBlockModeConfig, new SyncDescriptor(ConfigBlockModeConfig)); + serviceCollection.set(ICompletionsTelemetryService, new SyncDescriptor(CompletionsTelemetryServiceBridge)); + serviceCollection.set(ICompletionsTelemetryUserConfigService, new SyncDescriptor(TelemetryUserConfig)); + serviceCollection.set(ICompletionsRecentEditsProviderService, new SyncDescriptor(FullRecentEditsProvider, [undefined])); + serviceCollection.set(ICompletionsNotifierService, new SyncDescriptor(CompletionNotifier)); + serviceCollection.set(ICompletionsOpenAIFetcherService, new SyncDescriptor(LiveOpenAIFetcher)); + serviceCollection.set(ICompletionsModelManagerService, new SyncDescriptor(AvailableModelsManager, [true])); + serviceCollection.set(ICompletionsAsyncManagerService, new SyncDescriptor(AsyncCompletionManager)); + serviceCollection.set(ICompletionsContextProviderBridgeService, new SyncDescriptor(ContextProviderBridge)); + serviceCollection.set(ICompletionsUserErrorNotifierService, new SyncDescriptor(UserErrorNotifier)); + serviceCollection.set(ICompletionsRelatedFilesProviderService, new SyncDescriptor(CompositeRelatedFilesProvider)); + serviceCollection.set(ICompletionsFileSystemService, extensionFileSystem); + serviceCollection.set(ICompletionsContextProviderRegistryService, new SyncDescriptor(CachedContextProviderRegistry, [CoreContextProviderRegistry, contextProviderMatch])); + serviceCollection.set(ICompletionsPromiseQueueService, new PromiseQueue()); + serviceCollection.set(ICompletionsCitationManager, new SyncDescriptor(LoggingCitationManager)); + serviceCollection.set(ICompletionsContextProviderService, new ContextProviderStatistics()); + serviceCollection.set(ICompletionsPromptFactoryService, new SyncDescriptor(CompletionsPromptFactory)); + serviceCollection.set(ICompletionsFetcherService, new SyncDescriptor(CompletionsFetcher)); + serviceCollection.set(ICompletionsDefaultContextProviders, new DefaultContextProvidersContainer()); + + return serviceAccessor.get(IInstantiationService).createChild(serviceCollection, store); +} + +/** @public */ +export function setup(serviceAccessor: ServicesAccessor, disposables: DisposableStore) { + // This must be registered before activation! + // CodeQuote needs to listen for the initial token notification event. + disposables.add(serviceAccessor.get(ICompletionsCitationManager).register()); + + // Register to listen for changes to the active document to keep track + // of last access time + disposables.add(registerDocumentTracker(serviceAccessor)); + + // Register the context providers enabled by default. + const defaultContextProviders = serviceAccessor.get(ICompletionsDefaultContextProviders); + defaultContextProviders.add('ms-vscode.cpptools'); + defaultContextProviders.add('promptfile-ai-context-provider'); + + disposables.add(setupCompletionsExperimentationService(serviceAccessor)); +} + +export function registerUnificationCommands(accessor: ServicesAccessor): IDisposable { + const disposables = new DisposableStore(); + + disposables.add(registerEnablementCommands(accessor)); + disposables.add(registerStatusBar(accessor)); + disposables.add(registerDiagnosticCommands(accessor)); + disposables.add(registerPanelSupport(accessor)); + disposables.add(registerModelPickerCommands(accessor)); + + return disposables; +} + +function registerEnablementCommands(accessor: ServicesAccessor): IDisposable { + const disposables = new DisposableStore(); + const instantiationService = accessor.get(IInstantiationService); + + // Enable/Disable/Toggle completions commands [with Command Palette support] + function enable(id: string): IDisposable { + return registerCommandWrapper(accessor, id, async () => { + await instantiationService.invokeFunction(enableCompletions); + }); + } + function disable(id: string): IDisposable { + return registerCommandWrapper(accessor, id, async () => { + await instantiationService.invokeFunction(disableCompletions); + }); + } + function toggle(id: string): IDisposable { + return registerCommandWrapper(accessor, id, async () => { + await instantiationService.invokeFunction(toggleCompletions); + }); + } + + // To support command palette + disposables.add(enable(CMDEnableCompletionsChat)); + disposables.add(disable(CMDDisableCompletionsChat)); + disposables.add(toggle(CMDToggleCompletionsChat)); + + // To support keybindings/main functionality + disposables.add(enable(CMDEnableCompletionsClient)); + disposables.add(disable(CMDDisableCompletionsClient)); + disposables.add(toggle(CMDToggleCompletionsClient)); + + return disposables; +} + +function registerModelPickerCommands(accessor: ServicesAccessor): IDisposable { + const disposables = new DisposableStore(); + + const instantiationService = accessor.get(IInstantiationService); + + const modelsPicker = instantiationService.createInstance(ModelPickerManager); + + function registerModelPicker(commandId: string): IDisposable { + return registerCommandWrapper(accessor, commandId, async () => { + await modelsPicker.showModelPicker(); + }); + } + + // Model picker command [with Command Palette support] + disposables.add(registerModelPicker(CMDOpenModelPickerClient)); + disposables.add(registerModelPicker(CMDOpenModelPickerChat)); + + return disposables; +} + +function registerStatusBar(accessor: ServicesAccessor): IDisposable { + const disposables = new DisposableStore(); + + const instantiationService = accessor.get(IInstantiationService); + const copilotTokenManagerService = accessor.get(ICompletionsCopilotTokenManager); + const extensionStatusService = accessor.get(ICompletionsExtensionStatus); + + // Status menu command [with Command Palette support] + function registerStatusMenu(menuId: string): IDisposable { + return registerCommandWrapper(accessor, menuId, async () => { + if (extensionStatusService.kind === 'Error') { + // Try for a fresh token to clear up the error, but don't block the UI for too long. + await Promise.race([ + copilotTokenManagerService.primeToken(), + new Promise(resolve => setTimeout(resolve, 100)), + ]); + } + instantiationService.createInstance(CopilotStatusBarPickMenu).showStatusMenu(); + }); + } + disposables.add(registerStatusMenu(CMDToggleStatusMenuClient)); + disposables.add(registerStatusMenu(CMDToggleStatusMenuChat)); + + return disposables; +} + +function registerDiagnosticCommands(accessor: ServicesAccessor): IDisposable { + const disposables = new DisposableStore(); + + disposables.add(registerCommandWrapper(accessor, CMDOpenDocumentationClient, () => { + return env.openExternal( + URI.parse('https://docs.github.com/en/copilot/getting-started-with-github-copilot?tool=vscode') + ); + })); + disposables.add(registerCommandWrapper(accessor, CMDOpenLogsClient, () => { + outputChannel.show(); + })); + + return disposables; +} + +export function registerCommandWrapper(accessor: ServicesAccessor, command: string, fn: (...args: unknown[]) => unknown): IDisposable { + const instantiationService = accessor.get(IInstantiationService); + return commands.registerCommand(command, async (...args: unknown[]) => { + try { + await fn(...args); + } catch (error) { + instantiationService.invokeFunction(exception, error, command); + } + }); +} diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/citationManager.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/citationManager.ts new file mode 100644 index 0000000..0fe6b35 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/citationManager.ts @@ -0,0 +1,71 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { commands } from 'vscode'; +import { CodeReference } from '.'; +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { Disposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { onCopilotToken } from '../../../lib/src/auth/copilotTokenNotifier'; +import { ICompletionsCitationManager, IPDocumentCitation } from '../../../lib/src/citationManager'; +import { OutputPaneShowCommand } from '../../../lib/src/snippy/constants'; +import { copilotOutputLogTelemetry } from '../../../lib/src/snippy/telemetryHandlers'; +import { notify } from './matchNotifier'; +import { GitHubCopilotLogger } from './outputChannel'; + +/** + * Citation manager that logs citations to the VS Code log. On the first citation encountered, + * the user gets a notification. + */ +export class LoggingCitationManager extends Disposable implements ICompletionsCitationManager { + declare _serviceBrand: undefined; + + private logger?: GitHubCopilotLogger; + private readonly codeReference: CodeReference; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @IAuthenticationService authenticationService: IAuthenticationService, + ) { + super(); + this.codeReference = this._register(this.instantiationService.createInstance(CodeReference)); + const disposable = onCopilotToken(authenticationService, _ => { + if (this.logger) { + return; + } + this.logger = instantiationService.createInstance(GitHubCopilotLogger); + const initialNotificationCommand = commands.registerCommand(OutputPaneShowCommand, () => + this.logger?.forceShow() + ); + this.codeReference.addDisposable(initialNotificationCommand); + }); + this.codeReference.addDisposable(disposable); + } + + register() { + return this.codeReference.register(); + } + + async handleIPCodeCitation(citation: IPDocumentCitation): Promise { + if (!this.codeReference.enabled || !this.logger || citation.details.length === 0) { + return; + } + + const start = citation.location?.start; + const matchLocation = start ? `[Ln ${start.line + 1}, Col ${start.character + 1}]` : 'Location not available'; + const shortenedMatchText = `${citation.matchingText + ?.slice(0, 100) + .replace(/[\r\n\t]+|^[ \t]+/gm, ' ') + .trim()}...`; + + this.logger.info(citation.inDocumentUri, `Similar code at `, matchLocation, shortenedMatchText); + for (const detail of citation.details) { + const { license, url } = detail; + this.logger.info(`License: ${license.replace('NOASSERTION', 'unknown')}, URL: ${url}`); + } + copilotOutputLogTelemetry.handleWrite({ instantiationService: this.instantiationService }); + await this.instantiationService.invokeFunction(notify); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/codeReferenceEngagementTracker.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/codeReferenceEngagementTracker.ts new file mode 100644 index 0000000..cb5434b --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/codeReferenceEngagementTracker.ts @@ -0,0 +1,49 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TextEditor, window } from 'vscode'; +import { Disposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { copilotOutputLogTelemetry } from '../../../lib/src/snippy/telemetryHandlers'; +import { citationsChannelName } from './outputChannel'; + +export class CodeRefEngagementTracker extends Disposable { + private activeLog = false; + + constructor(@IInstantiationService private instantiationService: IInstantiationService) { + super(); + this._register(window.onDidChangeActiveTextEditor((e) => this.onActiveEditorChange(e))); + this._register(window.onDidChangeVisibleTextEditors((e) => this.onVisibleEditorsChange(e))); + } + + onActiveEditorChange = (editor: TextEditor | undefined) => { + if (this.isOutputLog(editor)) { + copilotOutputLogTelemetry.handleFocus({ instantiationService: this.instantiationService }); + } + }; + + onVisibleEditorsChange = (currEditors: readonly TextEditor[]) => { + const copilotLog = currEditors.find(e => this.isOutputLog(e)); + + if (this.activeLog) { + if (!copilotLog) { + this.activeLog = false; + } + } else if (copilotLog) { + this.activeLog = true; + copilotOutputLogTelemetry.handleOpen({ instantiationService: this.instantiationService }); + } + }; + + get logVisible() { + return this.activeLog; + } + + private isOutputLog = (editor: TextEditor | undefined) => { + return ( + editor && editor.document.uri.scheme === 'output' && editor.document.uri.path.includes(citationsChannelName) + ); + }; +} diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/index.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/index.ts new file mode 100644 index 0000000..1846481 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/index.ts @@ -0,0 +1,61 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from 'vscode'; +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { IDisposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotToken } from '../../../lib/src/auth/copilotTokenManager'; +import { onCopilotToken } from '../../../lib/src/auth/copilotTokenNotifier'; +import { ICompletionsLogTargetService } from '../../../lib/src/logger'; +import { codeReferenceLogger } from '../../../lib/src/snippy/logger'; +import { ICompletionsRuntimeModeService } from '../../../lib/src/util/runtimeMode'; +import { CodeRefEngagementTracker } from './codeReferenceEngagementTracker'; + +export class CodeReference implements IDisposable { + subscriptions: Disposable | undefined; + event?: Disposable; + enabled: boolean = false; + + constructor( + @IInstantiationService private readonly _instantiationService: IInstantiationService, + @ICompletionsRuntimeModeService readonly _runtimeMode: ICompletionsRuntimeModeService, + @ICompletionsLogTargetService private readonly _logTarget: ICompletionsLogTargetService, + @IAuthenticationService private readonly _authenticationService: IAuthenticationService, + ) { } + + dispose() { + this.subscriptions?.dispose(); + this.event?.dispose(); + } + + register() { + if (!this._runtimeMode.isRunningInTest()) { + this.event = onCopilotToken(this._authenticationService, (t) => this.onCopilotToken(t)); + } + return this; + } + + addDisposable(disposable: Disposable) { + if (!this.subscriptions) { + this.subscriptions = Disposable.from(disposable); + } else { + this.subscriptions = Disposable.from(this.subscriptions, disposable); + } + } + + onCopilotToken = (token: Omit) => { + this.enabled = token.codeQuoteEnabled || false; + if (!token.codeQuoteEnabled) { + this.subscriptions?.dispose(); + this.subscriptions = undefined; + codeReferenceLogger.debug(this._logTarget, 'Public code references are disabled.'); + return; + } + + codeReferenceLogger.info(this._logTarget, 'Public code references are enabled.'); + this.addDisposable(this._instantiationService.createInstance(CodeRefEngagementTracker)); + }; +} diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/matchNotifier.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/matchNotifier.ts new file mode 100644 index 0000000..e6a8c65 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/matchNotifier.ts @@ -0,0 +1,58 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { commands, env, Uri } from 'vscode'; +import { IVSCodeExtensionContext } from '../../../../../../platform/extContext/common/extensionContext'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsNotificationSender } from '../../../lib/src/notificationSender'; +import { OutputPaneShowCommand } from '../../../lib/src/snippy/constants'; +import { matchNotificationTelemetry, TelemetryActor } from '../../../lib/src/snippy/telemetryHandlers'; + +const matchCodeMessage = + 'We found a reference to public code in a recent suggestion. To learn more about public code references, review the [documentation](https://aka.ms/github-copilot-match-public-code).'; +const MatchAction = 'View reference'; +const SettingAction = 'Change setting'; +const CodeReferenceKey = 'codeReference.notified'; + +/** + * Displays a toast notification when the first code reference is found. + * The user will only ever see a single notification of this behavior. + * Displays the output panel on notification ack. + */ +export function notify(accessor: ServicesAccessor) { + const extension = accessor.get(IVSCodeExtensionContext); + const instantiationService = accessor.get(IInstantiationService); + const didNotify = extension.globalState.get(CodeReferenceKey); + + if (didNotify) { + return; + } + + const notificationSender = accessor.get(ICompletionsNotificationSender); + + const messageItems = [{ title: MatchAction }, { title: SettingAction }]; + + void notificationSender.showWarningMessage(matchCodeMessage, ...messageItems).then(async action => { + const event = { instantiationService, actor: 'user' as TelemetryActor }; + + switch (action?.title) { + case MatchAction: { + matchNotificationTelemetry.handleDoAction(event); + await commands.executeCommand(OutputPaneShowCommand); + break; + } + case SettingAction: { + await env.openExternal(Uri.parse('https://aka.ms/github-copilot-settings')); + break; + } + case undefined: { + matchNotificationTelemetry.handleDismiss(event); + break; + } + } + }); + + return extension.globalState.update(CodeReferenceKey, true); +} diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/outputChannel.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/outputChannel.ts new file mode 100644 index 0000000..0360f4b --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/outputChannel.ts @@ -0,0 +1,102 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { window, type OutputChannel } from 'vscode'; +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { Disposable, IDisposable, MutableDisposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotToken } from '../../../lib/src/auth/copilotTokenManager'; +import { onCopilotToken } from '../../../lib/src/auth/copilotTokenNotifier'; + +interface GitHubLogger extends Disposable { + info(...messages: string[]): void; + forceShow(): void; +} + +export const citationsChannelName = 'GitHub Copilot Log (Code References)'; + +// Literally taken from VS Code +function getCurrentTimestamp() { + const toTwoDigits = (v: number) => (v < 10 ? `0${v}` : v); + const toThreeDigits = (v: number) => (v < 10 ? `00${v}` : v < 100 ? `0${v}` : v); + const currentTime = new Date(); + return `${currentTime.getFullYear()}-${toTwoDigits(currentTime.getMonth() + 1)}-${toTwoDigits( + currentTime.getDate() + )} ${toTwoDigits(currentTime.getHours())}:${toTwoDigits(currentTime.getMinutes())}:${toTwoDigits( + currentTime.getSeconds() + )}.${toThreeDigits(currentTime.getMilliseconds())}`; +} + +class CodeReferenceOutputChannel implements IDisposable { + constructor(private output: OutputChannel) { } + + info(...messages: string[]) { + this.output.appendLine(`${getCurrentTimestamp()} [info] ${messages.join(' ')}`); + } + + show(preserveFocus: boolean) { + this.output.show(preserveFocus); + } + + dispose() { + this.output.dispose(); + } +} + +export class GitHubCopilotLogger extends Disposable implements GitHubLogger { + + private output = this._register(new MutableDisposable()); + + constructor( + @IInstantiationService instantiationService: IInstantiationService, + @IAuthenticationService authenticationService: IAuthenticationService + ) { + super(); + this._register(onCopilotToken(authenticationService, t => this.checkCopilotToken(t))); + + this.createChannel(); + } + + private checkCopilotToken = (token: Omit) => { + if (token.codeQuoteEnabled) { + this.createChannel(); + } else { + this.removeChannel(); + } + }; + + private log(type: 'info', ...messages: string[]) { + const output = this.createChannel(); + + const [base, ...rest] = messages; + output[type](base, ...rest); + } + + info(...messages: string[]) { + this.log('info', ...messages); + } + + forceShow() { + // Preserve focus in the editor + this.getChannel()?.show(true); + } + + private createChannel(): CodeReferenceOutputChannel { + if (this.output.value) { + return this.output.value; + } + + this.output.value = new CodeReferenceOutputChannel(window.createOutputChannel(citationsChannelName, 'code-referencing')); + return this.output.value; + } + + private getChannel(): CodeReferenceOutputChannel | undefined { + return this.output.value; + } + + private removeChannel() { + this.output.value = undefined; + } +} diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/test/codeReferenceEngagementTracker.test.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/test/codeReferenceEngagementTracker.test.ts new file mode 100644 index 0000000..135f626 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/test/codeReferenceEngagementTracker.test.ts @@ -0,0 +1,101 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import { TextEditor } from 'vscode'; +import { DisposableStore } from '../../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { withInMemoryTelemetry } from '../../../../lib/src/test/telemetry'; +import { createExtensionTestingContext } from '../../test/context'; +import { CodeRefEngagementTracker } from '../codeReferenceEngagementTracker'; +import { citationsChannelName } from '../outputChannel'; + +suite('CodeReferenceEngagementTracker', function () { + let engagementTracker: CodeRefEngagementTracker; + let accessor: ServicesAccessor; + const disposables = new DisposableStore(); + + setup(function () { + accessor = createExtensionTestingContext().createTestingAccessor(); + engagementTracker = disposables.add(accessor.get(IInstantiationService).createInstance(CodeRefEngagementTracker)); + }); + + teardown(function () { + disposables.clear(); + }); + + test('sends a telemetry event when the output channel is focused', async function () { + const telemetry = await withInMemoryTelemetry(accessor, () => { + engagementTracker.onActiveEditorChange({ + document: { uri: { scheme: 'output', path: citationsChannelName } }, + } as TextEditor); + }); + + assert.ok(telemetry.reporter.events.length === 1); + assert.strictEqual(telemetry.reporter.events[0].name, 'code_referencing.github_copilot_log.focus.count'); + }); + + test('sends a telemetry event when the output channel is focused2', async function () { + const telemetry = await withInMemoryTelemetry(accessor, () => { + engagementTracker.onActiveEditorChange({ + document: { uri: { scheme: 'output', path: citationsChannelName } }, + } as TextEditor); + }); + + assert.ok(telemetry.reporter.events.length === 1); + assert.strictEqual(telemetry.reporter.events[0].name, 'code_referencing.github_copilot_log.focus.count'); + }); + + + test('sends a telemetry event when the output channel is opened', async function () { + const telemetry = await withInMemoryTelemetry(accessor, () => { + engagementTracker.onVisibleEditorsChange([ + { + document: { uri: { scheme: 'output', path: citationsChannelName } }, + }, + ] as TextEditor[]); + }); + + assert.ok(telemetry.reporter.events.length === 1); + assert.strictEqual(telemetry.reporter.events[0].name, 'code_referencing.github_copilot_log.open.count'); + }); + + test('does not send a telemetry event when the output channel is already opened', async function () { + const telemetry = await withInMemoryTelemetry(accessor, () => { + engagementTracker.onVisibleEditorsChange([ + { + document: { uri: { scheme: 'output', path: citationsChannelName } }, + }, + ] as TextEditor[]); + engagementTracker.onVisibleEditorsChange([ + { + document: { uri: { scheme: 'output', path: citationsChannelName } }, + }, + { + document: { uri: { scheme: 'file', path: 'some-other-file.js' } }, + }, + ] as TextEditor[]); + }); + + assert.ok(telemetry.reporter.events.length === 1); + }); + + test('tracks when the log closes internally', async function () { + const telemetry = await withInMemoryTelemetry(accessor, () => { + engagementTracker.onVisibleEditorsChange([ + { + document: { uri: { scheme: 'output', path: citationsChannelName } }, + }, + ] as TextEditor[]); + engagementTracker.onVisibleEditorsChange([ + { + document: { uri: { scheme: 'file', path: 'some-other-file.js' } }, + }, + ] as TextEditor[]); + }); + + assert.ok(telemetry.reporter.events.length === 1); + assert.ok(engagementTracker.logVisible === false); + }); +}); diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/test/codeReferencing.test.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/test/codeReferencing.test.ts new file mode 100644 index 0000000..ae85fff --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/test/codeReferencing.test.ts @@ -0,0 +1,70 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import * as Sinon from 'sinon'; +import { Disposable, ExtensionContext } from 'vscode'; +import { CodeReference } from '..'; +import { CopilotToken, createTestExtendedTokenInfo } from '../../../../../../../platform/authentication/common/copilotToken'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ConnectionState } from '../../../../lib/src/snippy/connectionState'; +import { createExtensionTestingContext } from '../../test/context'; + +function testExtensionContext() { + return { + subscriptions: [], + }; +} + +suite('CodeReference', function () { + let extensionContext: ExtensionContext; + let instantiationService: IInstantiationService; + let sub: Disposable | undefined; + + setup(function () { + const accessor = createExtensionTestingContext().createTestingAccessor(); + instantiationService = accessor.get(IInstantiationService); + extensionContext = testExtensionContext() as unknown as ExtensionContext; + }); + + teardown(function () { + extensionContext.subscriptions.forEach(sub => { + sub.dispose(); + }); + sub?.dispose(); + ConnectionState.setDisabled(); + }); + + suite('subscriptions', function () { + test('should be undefined by default', function () { + const result = instantiationService.createInstance(CodeReference); + sub = result.subscriptions; + assert.ok(!sub); + }); + + test('should be updated correctly when token change events received', function () { + const codeQuote = instantiationService.createInstance(CodeReference); + const enabledToken = new CopilotToken(createTestExtendedTokenInfo({ token: `test token ${generateUuid()}`, username: 'fixedTokenManager', copilot_plan: 'unknown', code_quote_enabled: true })); + const disabledToken = new CopilotToken(createTestExtendedTokenInfo({ token: `test token ${generateUuid()}`, username: 'fixedTokenManager', copilot_plan: 'unknown', code_quote_enabled: false })); + + codeQuote.onCopilotToken(enabledToken); + + assert.ok(codeQuote.enabled); + assert.ok(codeQuote.subscriptions); + assert.ok(codeQuote.subscriptions instanceof Disposable); + + const subSpy = Sinon.spy(codeQuote.subscriptions, 'dispose'); + codeQuote.onCopilotToken(disabledToken); + + assert.ok(!codeQuote.enabled); + assert.strictEqual(codeQuote.subscriptions, undefined); + assert.strictEqual(subSpy.calledOnce, true); + + codeQuote.onCopilotToken(enabledToken); + assert.ok(codeQuote.enabled); + assert.notStrictEqual(codeQuote.subscriptions, undefined); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/extension/src/codeReferencing/test/matchNotifier.test.ts b/completions-sample-code/vscode-node/extension/src/codeReferencing/test/matchNotifier.test.ts new file mode 100644 index 0000000..f13858e --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/codeReferencing/test/matchNotifier.test.ts @@ -0,0 +1,121 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import sinon from 'sinon'; +import { commands, env } from 'vscode'; +import { IVSCodeExtensionContext } from '../../../../../../../platform/extContext/common/extensionContext'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsNotificationSender } from '../../../../lib/src/notificationSender'; +import { OutputPaneShowCommand } from '../../../../lib/src/snippy/constants'; +import { withInMemoryTelemetry } from '../../../../lib/src/test/telemetry'; +import { TestNotificationSender } from '../../../../lib/src/test/testHelpers'; +import { createExtensionTestingContext } from '../../test/context'; +import { notify } from '../matchNotifier'; + +suite('.match', function () { + let accessor: ServicesAccessor; + + setup(function () { + accessor = createExtensionTestingContext().createTestingAccessor(); + }); + + test('populates the globalState object', async function () { + const extensionContext = accessor.get(IVSCodeExtensionContext); + const globalState = extensionContext.globalState; + + await notify(accessor); + + assert.ok(globalState.get('codeReference.notified')); + }); + + test('notifies the user', async function () { + const testNotificationSender = accessor.get(ICompletionsNotificationSender) as TestNotificationSender; + testNotificationSender.performAction('View reference'); + + await notify(accessor); + + assert.strictEqual(testNotificationSender.sentMessages.length, 1); + }); + + test('sends a telemetry event on view reference action', async function () { + const testNotificationSender = accessor.get(ICompletionsNotificationSender) as TestNotificationSender; + testNotificationSender.performAction('View reference'); + + const telemetry = await withInMemoryTelemetry(accessor, async accessor => { + await notify(accessor); + }); + + assert.strictEqual(telemetry.reporter.events.length, 1); + assert.strictEqual(telemetry.reporter.events[0].name, 'code_referencing.match_notification.acknowledge.count'); + }); + + test('executes the output panel display command on view reference action', async function () { + const spy = sinon.spy(commands, 'executeCommand'); + const testNotificationSender = accessor.get(ICompletionsNotificationSender) as TestNotificationSender; + testNotificationSender.performAction('View reference'); + + await notify(accessor); + + await testNotificationSender.waitForMessages(); + + assert.ok(spy.calledOnce); + assert.ok(spy.calledWith(OutputPaneShowCommand)); + + spy.restore(); + }); + + test('opens the settings page on change setting action', async function () { + const stub = sinon.stub(env, 'openExternal'); + const testNotificationSender = accessor.get(ICompletionsNotificationSender) as TestNotificationSender; + testNotificationSender.performAction('Change setting'); + + await notify(accessor); + + await testNotificationSender.waitForMessages(); + + assert.ok(stub.calledOnce); + assert.ok( + stub.calledWith( + sinon.match({ + scheme: 'https', + authority: 'aka.ms', + path: '/github-copilot-settings', + }) + ) + ); + + stub.restore(); + }); + + test('sends a telemetry event on notification dismissal', async function () { + const testNotificationSender = accessor.get(ICompletionsNotificationSender) as TestNotificationSender; + testNotificationSender.performDismiss(); + + const telemetry = await withInMemoryTelemetry(accessor, async accessor => { + await notify(accessor); + }); + + await testNotificationSender.waitForMessages(); + + assert.strictEqual(telemetry.reporter.events.length, 1); + assert.strictEqual(telemetry.reporter.events[0].name, 'code_referencing.match_notification.ignore.count'); + }); + + test('does not notify if already notified', async function () { + const extensionContext = accessor.get(IVSCodeExtensionContext); + const instantiationService = accessor.get(IInstantiationService); + const globalState = extensionContext.globalState; + const testNotificationSender = accessor.get(ICompletionsNotificationSender) as TestNotificationSender; + testNotificationSender.performAction('View reference'); + + await globalState.update('codeReference.notified', true); + + await instantiationService.invokeFunction(notify); + + await testNotificationSender.waitForMessages(); + + assert.strictEqual(testNotificationSender.sentMessages.length, 0); + }); +}); diff --git a/completions-sample-code/vscode-node/extension/src/completionsObservableWorkspace.ts b/completions-sample-code/vscode-node/extension/src/completionsObservableWorkspace.ts new file mode 100644 index 0000000..5fe6ef4 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/completionsObservableWorkspace.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { VSCodeWorkspace } from '../../../../inlineEdits/vscode-node/parts/vscodeWorkspace'; +import { ICompletionsObservableWorkspace } from '../../lib/src/completionsObservableWorkspace'; + +export class CompletionsObservableWorkspace extends VSCodeWorkspace implements ICompletionsObservableWorkspace { + declare _serviceBrand: undefined; +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/extension/src/config.ts b/completions-sample-code/vscode-node/extension/src/config.ts new file mode 100644 index 0000000..af0aef0 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/config.ts @@ -0,0 +1,249 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { WorkspaceConfiguration } from 'vscode'; +import * as vscode from 'vscode'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + ConfigKey, + ConfigKeyType, + ConfigProvider, getConfigDefaultForKey, + getConfigKeyRecursively, + getOptionalConfigDefaultForKey, + ICompletionsConfigProvider, + ICompletionsEditorAndPluginInfo, + packageJson +} from '../../lib/src/config'; +import { CopilotConfigPrefix } from '../../lib/src/constants'; +import { Logger } from '../../lib/src/logger'; +import { transformEvent } from '../../lib/src/util/event'; + +const logger = new Logger('extensionConfig'); + +export class VSCodeConfigProvider extends ConfigProvider { + private config: WorkspaceConfiguration; + + constructor() { + super(); + this.config = vscode.workspace.getConfiguration(CopilotConfigPrefix); + + // Reload cached config if a workspace config change effects Copilot namespace + vscode.workspace.onDidChangeConfiguration(changeEvent => { + if (changeEvent.affectsConfiguration(CopilotConfigPrefix)) { + this.config = vscode.workspace.getConfiguration(CopilotConfigPrefix); + } + }); + } + + override getConfig(key: ConfigKeyType): T { + return getConfigKeyRecursively(this.config, key) ?? getConfigDefaultForKey(key); + } + + override getOptionalConfig(key: ConfigKeyType): T | undefined { + return getConfigKeyRecursively(this.config, key) ?? getOptionalConfigDefaultForKey(key); + } + + // Dumps config settings defined in the extension json + override dumpForTelemetry(): { [key: string]: string } { + return {}; + } + + override onDidChangeCopilotSettings: ConfigProvider['onDidChangeCopilotSettings'] = transformEvent( + vscode.workspace.onDidChangeConfiguration, + event => { + if (event.affectsConfiguration('github.copilot')) { + return this; + } + if (event.affectsConfiguration('github.copilot-chat')) { + return this; + } + } + ); +} + +// From vscode's src/vs/platform/telemetry/common/telemetryUtils.ts +const telemetryAllowedAuthorities = new Set([ + 'ssh-remote', + 'dev-container', + 'attached-container', + 'wsl', + 'tunnel', + 'codespaces', + 'amlext', +]); + +export class VSCodeEditorInfo implements ICompletionsEditorAndPluginInfo { + declare _serviceBrand: undefined; + getEditorInfo() { + let devName = vscode.env.uriScheme; + if (vscode.version.endsWith('-insider')) { + devName = devName.replace(/-insiders$/, ''); + } + const remoteName = vscode.env.remoteName; + if (remoteName) { + devName += `@${telemetryAllowedAuthorities.has(remoteName) ? remoteName : 'other'}`; + } + return { + name: 'vscode', + readableName: vscode.env.appName.replace(/ - Insiders$/, ''), + devName: devName, + version: vscode.version, + root: vscode.env.appRoot, + }; + } + getEditorPluginInfo() { + return { name: 'copilot-chat', readableName: 'GitHub Copilot for Visual Studio Code', version: packageJson.version }; + } + getRelatedPluginInfo() { + // Any additions to this list should also be added as a known filter in + // lib/src/experiments/filters.ts + return [ + 'ms-vscode.cpptools', + 'ms-vscode.cmake-tools', + 'ms-vscode.makefile-tools', + 'ms-dotnettools.csdevkit', + 'ms-python.python', + 'ms-python.vscode-pylance', + 'vscjava.vscode-java-pack', + 'vscjava.vscode-java-dependency', + 'vscode.typescript-language-features', + 'ms-vscode.vscode-typescript-next', + 'ms-dotnettools.csharp', + 'github.copilot-chat', + ] + .map(name => { + const extpj = vscode.extensions.getExtension(name)?.packageJSON as unknown; + if (extpj && typeof extpj === 'object' && 'version' in extpj && typeof extpj.version === 'string') { + return { name, version: extpj.version }; + } + }) + .filter(plugin => plugin !== undefined); + } +} + +type EnabledConfigKeyType = { [key: string]: boolean }; + +function getEnabledConfigObject(accessor: ServicesAccessor): EnabledConfigKeyType { + const configProvider = accessor.get(ICompletionsConfigProvider); + return { '*': true, ...(configProvider.getConfig(ConfigKey.Enable) ?? {}) }; +} + +function getEnabledConfig(accessor: ServicesAccessor, languageId: string): boolean { + const obj = getEnabledConfigObject(accessor); + return obj[languageId] ?? obj['*'] ?? true; +} + +/** + * Checks if automatic completions are enabled for the current document by all Copilot completion settings. + * Excludes the `editor.inlineSuggest.enabled` setting. + * Return undefined if there is no current document. + */ +export function isCompletionEnabled(accessor: ServicesAccessor): boolean | undefined { + const editor = vscode.window.activeTextEditor; + if (!editor) { + return undefined; + } + return isCompletionEnabledForDocument(accessor, editor.document); +} + +export function isCompletionEnabledForDocument(accessor: ServicesAccessor, document: vscode.TextDocument): boolean { + return getEnabledConfig(accessor, document.languageId); +} + +export function isInlineSuggestEnabled(): boolean | undefined { + return vscode.workspace.getConfiguration('editor.inlineSuggest').get('enabled'); +} + +type ConfigurationInspect = Exclude, undefined>; +const inspectKinds: [keyof ConfigurationInspect, vscode.ConfigurationTarget, boolean][] = [ + ['workspaceFolderLanguageValue', vscode.ConfigurationTarget.WorkspaceFolder, true], + ['workspaceFolderValue', vscode.ConfigurationTarget.WorkspaceFolder, false], + ['workspaceLanguageValue', vscode.ConfigurationTarget.Workspace, true], + ['workspaceValue', vscode.ConfigurationTarget.Workspace, false], + ['globalLanguageValue', vscode.ConfigurationTarget.Global, true], + ['globalValue', vscode.ConfigurationTarget.Global, false], +]; + +function getConfigurationTargetForEnabledConfig(): vscode.ConfigurationTarget { + const inspect = vscode.workspace.getConfiguration(CopilotConfigPrefix).inspect(ConfigKey.Enable); + if (inspect?.workspaceFolderValue !== undefined) { + return vscode.ConfigurationTarget.WorkspaceFolder; + } else if (inspect?.workspaceValue !== undefined) { + return vscode.ConfigurationTarget.Workspace; + } else { + return vscode.ConfigurationTarget.Global; + } +} + +/** + * Enable completions by every means possible. + */ +export async function enableCompletions(accessor: ServicesAccessor) { + const instantiationService = accessor.get(IInstantiationService); + const scope = vscode.window.activeTextEditor?.document; + // Make sure both of these settings are enabled, because that's a precondition for the user seeing inline completions. + for (const [section, option] of [['', 'editor.inlineSuggest.enabled']]) { + const config = vscode.workspace.getConfiguration(section, scope); + const inspect = config.inspect(option); + // Start from the most specific setting and work our way up to the global default. + for (const [key, target, overrideInLanguage] of inspectKinds) { + // Exit condition: if VS Code thinks the setting is enabled, we're done. + // This might be true from the start, or a call to .update() might flip it. + if (vscode.workspace.getConfiguration(section, scope).get(option)) { + break; + } + if (inspect?.[key] === false) { + await config.update(option, true, target, overrideInLanguage); + } + } + } + + // The rest of this function is the inverse of disableCompletions(), updating the github.copilot.enable setting. + const languageId = vscode.window.activeTextEditor?.document.languageId; + if (!languageId) { return; } + const config = vscode.workspace.getConfiguration(CopilotConfigPrefix); + const enabledConfig = { ...instantiationService.invokeFunction(getEnabledConfigObject) }; + if (!(languageId in enabledConfig)) { + enabledConfig['*'] = true; + } else { + enabledConfig[languageId] = true; + } + await config.update(ConfigKey.Enable, enabledConfig, getConfigurationTargetForEnabledConfig()); + if (!instantiationService.invokeFunction(isCompletionEnabled)) { + const inspect = vscode.workspace.getConfiguration(CopilotConfigPrefix).inspect(ConfigKey.Enable); + const error = new Error(`Failed to enable completions for ${languageId}: ${JSON.stringify(inspect)}`); + instantiationService.invokeFunction(acc => logger.exception(acc, error, '.enable')); + } +} + +/** + * Disable completions using the github.copilot.enable setting. + */ +export async function disableCompletions(accessor: ServicesAccessor) { + const instantiationService = accessor.get(IInstantiationService); + const languageId = vscode.window.activeTextEditor?.document.languageId; + if (!languageId) { return; } + const config = vscode.workspace.getConfiguration(CopilotConfigPrefix); + const enabledConfig = { ...instantiationService.invokeFunction(getEnabledConfigObject) }; + if (!(languageId in enabledConfig)) { + enabledConfig['*'] = false; + } else if (enabledConfig[languageId]) { + enabledConfig[languageId] = false; + } + await config.update(ConfigKey.Enable, enabledConfig, getConfigurationTargetForEnabledConfig()); + if (instantiationService.invokeFunction(isCompletionEnabled)) { + const inspect = vscode.workspace.getConfiguration(CopilotConfigPrefix).inspect(ConfigKey.Enable); + const error = new Error(`Failed to disable completions for ${languageId}: ${JSON.stringify(inspect)}`); + instantiationService.invokeFunction(acc => logger.exception(acc, error, '.disable')); + } +} + +export async function toggleCompletions(accessor: ServicesAccessor) { + if (isCompletionEnabled(accessor) && isInlineSuggestEnabled()) { + await disableCompletions(accessor); + } else { + await enableCompletions(accessor); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/constants.ts b/completions-sample-code/vscode-node/extension/src/constants.ts new file mode 100644 index 0000000..9891762 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/constants.ts @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Commands ending with "Client" refer to the command ID used in the legacy Copilot extension. +// - These IDs should not appear in the package.json file +// - These IDs should be registered to support all functionality (except if this command needs to be supported when both extensions are loaded/active). +// Commands ending with "Chat" refer to the command ID used in the Copilot Chat extension. +// - These IDs should be used in package.json +// - These IDs should only be registered if they appear in the package.json (meaning the command palette) or if the command needs to be supported when both extensions are loaded/active. + +export const CMDOpenPanelClient = 'github.copilot.generate'; +export const CMDOpenPanelChat = 'github.copilot.chat.openSuggestionsPanel'; // "github.copilot.chat.generate" is already being used + +export const CMDAcceptCursorPanelSolutionClient = 'github.copilot.acceptCursorPanelSolution'; +export const CMDNavigatePreviousPanelSolutionClient = 'github.copilot.previousPanelSolution'; +export const CMDNavigateNextPanelSolutionClient = 'github.copilot.nextPanelSolution'; + +export const CMDToggleStatusMenuClient = 'github.copilot.toggleStatusMenu'; +export const CMDToggleStatusMenuChat = 'github.copilot.chat.toggleStatusMenu'; + +// Needs to be supported in both extensions when they are loaded/active. Requires a different ID. +export const CMDSendCompletionsFeedbackChat = 'github.copilot.chat.sendCompletionFeedback'; + +export const CMDEnableCompletionsChat = 'github.copilot.chat.completions.enable'; +export const CMDDisableCompletionsChat = 'github.copilot.chat.completions.disable'; +export const CMDToggleCompletionsChat = 'github.copilot.chat.completions.toggle'; +export const CMDEnableCompletionsClient = 'github.copilot.completions.enable'; +export const CMDDisableCompletionsClient = 'github.copilot.completions.disable'; +export const CMDToggleCompletionsClient = 'github.copilot.completions.toggle'; + +export const CMDOpenLogsClient = 'github.copilot.openLogs'; +export const CMDOpenDocumentationClient = 'github.copilot.openDocs'; + +// Existing chat command reused for diagnostics +export const CMDCollectDiagnosticsChat = 'github.copilot.debug.collectDiagnostics'; + +// Context variable that enable/disable panel-specific commands +export const CopilotPanelVisible = 'github.copilot.panelVisible'; +export const ComparisonPanelVisible = 'github.copilot.comparisonPanelVisible'; + +export const CMDOpenModelPickerClient = 'github.copilot.openModelPicker'; +export const CMDOpenModelPickerChat = 'github.copilot.chat.openModelPicker'; \ No newline at end of file diff --git a/completions-sample-code/vscode-node/extension/src/contextProviderMatch.ts b/completions-sample-code/vscode-node/extension/src/contextProviderMatch.ts new file mode 100644 index 0000000..d80f6b6 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/contextProviderMatch.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { languages, workspace } from 'vscode'; +import { DocumentSelector } from 'vscode-languageserver-protocol'; +import { IInstantiationService } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { isDocumentValid } from '../../lib/src/util/documentEvaluation'; +import { DocumentContext } from '../../types/src'; + +export async function contextProviderMatch( + instantiationService: IInstantiationService, + documentSelector: DocumentSelector, + documentContext: DocumentContext +): Promise { + const vscDoc = workspace.textDocuments.find(td => td.uri.toString() === documentContext.uri); + if (!vscDoc) { + return 0; + } + + const result = await instantiationService.invokeFunction(isDocumentValid, documentContext); + if (result.status !== 'valid') { + return 0; + } + + return languages.match(documentSelector, vscDoc); +} diff --git a/completions-sample-code/vscode-node/extension/src/copilotCompletionFeedbackTracker.ts b/completions-sample-code/vscode-node/extension/src/copilotCompletionFeedbackTracker.ts new file mode 100644 index 0000000..3f1753b --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotCompletionFeedbackTracker.ts @@ -0,0 +1,83 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Command, commands, InlineCompletionItem, Uri } from 'vscode'; +import { Disposable } from '../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { collectCompletionDiagnostics, formatDiagnosticsAsMarkdown } from '../../lib/src/diagnostics'; +import { telemetry, TelemetryData } from '../../lib/src/telemetry'; +import { CMDSendCompletionsFeedbackChat } from './constants'; + +export const sendCompletionFeedbackCommand: Command = { + command: CMDSendCompletionsFeedbackChat, + title: 'Send Copilot Completion Feedback', + tooltip: 'Send feedback about the last shown Copilot completion item', +}; + +export class CopilotCompletionFeedbackTracker extends Disposable { + private lastShownCopilotCompletionItem: InlineCompletionItem | undefined; + + constructor(@IInstantiationService private readonly instantiationService: IInstantiationService) { + super(); + this._register(commands.registerCommand(sendCompletionFeedbackCommand.command, async () => { + const commandArg: unknown = this.lastShownCopilotCompletionItem?.command?.arguments?.[0]; + let telemetryArg: TelemetryData | undefined; + if (commandArg && typeof commandArg === 'object' && 'telemetry' in commandArg) { + if (commandArg.telemetry instanceof TelemetryData) { + telemetryArg = commandArg.telemetry; + } + } + this.instantiationService.invokeFunction(telemetry, 'ghostText.sentFeedback', telemetryArg); + + await this.instantiationService.invokeFunction(openGitHubIssue, this.lastShownCopilotCompletionItem, telemetryArg); + })); + } + + trackItem(item: InlineCompletionItem) { + this.lastShownCopilotCompletionItem = item; + } +} + +async function openGitHubIssue( + accessor: ServicesAccessor, + item: InlineCompletionItem | undefined, + telemetry: TelemetryData | undefined +) { + const body = generateGitHubIssueBody(accessor, item, telemetry); + await commands.executeCommand('workbench.action.openIssueReporter', { + extensionId: 'github.copilot', + uri: Uri.parse('https://github.com/microsoft/vscode'), + data: body, + }); +} + +function generateGitHubIssueBody( + accessor: ServicesAccessor, + item: InlineCompletionItem | undefined, + telemetry: TelemetryData | undefined +) { + const diagnostics = collectCompletionDiagnostics(accessor, telemetry); + const formattedDiagnostics = formatDiagnosticsAsMarkdown(diagnostics); + if (typeof item?.insertText !== 'string') { + return ''; + } + + return `## Copilot Completion Feedback +### Describe the issue, feedback, or steps to reproduce it: + + +### Completion text: +\`\`\` +${item.insertText} +\`\`\` + +
+Diagnostics + +${formattedDiagnostics} + +
+`; +} diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/common.ts b/completions-sample-code/vscode-node/extension/src/copilotPanel/common.ts new file mode 100644 index 0000000..9949f14 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/common.ts @@ -0,0 +1,65 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Range, commands, window, type Disposable } from 'vscode'; +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { DisposableStore, IDisposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService, type ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import * as constants from '../constants'; +import { registerCommand } from '../telemetry'; +import { wrapDoc } from '../textDocumentManager'; +import { CopilotSuggestionsPanelManager } from './copilotSuggestionsPanelManager'; + +// Exported for testing +export enum PanelNavigationType { + Previous = 'previous', + Next = 'next', +} + +/** + * This interface contains data associated to a completion displayed in the panel. + */ +export interface PanelCompletion { + insertText: string; + range: Range; + copilotAnnotations?: CopilotNamedAnnotationList; + postInsertionCallback: () => PromiseLike | void; +} + +export function registerPanelSupport(accessor: ServicesAccessor): Disposable { + const instantiationService = accessor.get(IInstantiationService); + const suggestionsPanelManager = instantiationService.createInstance(CopilotSuggestionsPanelManager); + + const disposableStore = new DisposableStore(); + + function registerOpenPanelCommand(id: string): IDisposable { + return registerCommand(accessor, id, async () => { + // hide ghost text while opening the generation ui + await commands.executeCommand('editor.action.inlineSuggest.hide'); + await instantiationService.invokeFunction(commandOpenPanel, suggestionsPanelManager); + }); + } + + // Register both commands to also support command palette + disposableStore.add(registerOpenPanelCommand(constants.CMDOpenPanelChat)); + disposableStore.add(registerOpenPanelCommand(constants.CMDOpenPanelClient)); + + // No command palette support needed for these commands + disposableStore.add(suggestionsPanelManager.registerCommands()); + + return disposableStore; +} + +function commandOpenPanel(accessor: ServicesAccessor, suggestionsPanelManager: CopilotSuggestionsPanelManager) { + const editor = window.activeTextEditor; + if (!editor) { return; } + const wrapped = wrapDoc(editor.document); + if (!wrapped) { return; } + + const { line, character } = editor.selection.active; + + suggestionsPanelManager.renderPanel(editor.document, { line, character }, wrapped); + return commands.executeCommand('setContext', constants.CopilotPanelVisible, true); +} diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotListDocument.ts b/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotListDocument.ts new file mode 100644 index 0000000..e26c0cb --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotListDocument.ts @@ -0,0 +1,49 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { IPosition, ITextDocument } from '../../../lib/src/textDocument'; +import { solutionCountTarget } from '../lib/copilotPanel/common'; +import { runSolutions } from '../lib/copilotPanel/panel'; +import { UnformattedSolution } from '../lib/panelShared/panelTypes'; +import { BaseListDocument } from '../panelShared/baseListDocument'; +import { BasePanelCompletion, ISuggestionsPanel } from '../panelShared/basePanelTypes'; +import { PanelCompletion } from './common'; + +/** + * Class representing a Open Copilot list using a ITextDocument as a way of displaying results. + * Currently only used in the VSCode extension. + */ +export class CopilotListDocument extends BaseListDocument { + constructor( + textDocument: ITextDocument, + position: IPosition, + panel: ISuggestionsPanel, + countTarget = solutionCountTarget, + @IInstantiationService instantiationService: IInstantiationService + ) { + super(textDocument, position, panel, countTarget, instantiationService); + } + + protected createPanelCompletion( + unformatted: UnformattedSolution, + baseCompletion: BasePanelCompletion + ): PanelCompletion { + return { + insertText: baseCompletion.insertText, + range: baseCompletion.range, + copilotAnnotations: baseCompletion.copilotAnnotations, + postInsertionCallback: baseCompletion.postInsertionCallback, + }; + } + + protected shouldAddSolution(newItem: PanelCompletion): boolean { + return !this.findDuplicateSolution(newItem); + } + + protected runSolutionsImpl(): Promise { + return this.instantiationService.invokeFunction(runSolutions, this, this); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotSuggestionsPanel.ts b/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotSuggestionsPanel.ts new file mode 100644 index 0000000..6400b3a --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotSuggestionsPanel.ts @@ -0,0 +1,53 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TextDocument, WebviewPanel } from 'vscode'; +import { IVSCodeExtensionContext } from '../../../../../../platform/extContext/common/extensionContext'; +import { BaseSuggestionsPanel, SolutionContent, WebviewMessage } from '../panelShared/baseSuggestionsPanel'; +import { PanelCompletion } from './common'; +import { CopilotSuggestionsPanelManager } from './copilotSuggestionsPanelManager'; +import { copilotPanelConfig } from './panelConfig'; + +export interface CopilotSolutionsMessage { + command: 'solutionsUpdated'; + solutions: SolutionContent[]; + percentage: number; +} + +export class CopilotSuggestionsPanel extends BaseSuggestionsPanel { + constructor( + webviewPanel: WebviewPanel, + document: TextDocument, + suggestionsPanelManager: CopilotSuggestionsPanelManager, + @IVSCodeExtensionContext contextService: IVSCodeExtensionContext, + ) { + super(webviewPanel, document, suggestionsPanelManager, copilotPanelConfig, contextService); + } + + protected renderSolutionContent(item: PanelCompletion, baseContent: SolutionContent): SolutionContent { + // Copilot panel just returns the base content without modifications + return baseContent; + } + + protected createSolutionsMessage(content: SolutionContent[], percentage: number): CopilotSolutionsMessage { + return { + command: 'solutionsUpdated', + solutions: content, + percentage, + }; + } + + protected override async handleCustomMessage(message: WebviewMessage): Promise { + switch (message.command) { + case 'acceptSolution': { + const solution = this.items()[message.solutionIndex]; + await this.acceptSolution(solution, true); + return Promise.resolve(true); + } + default: + return Promise.resolve(false); + } + } +} diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotSuggestionsPanelManager.ts b/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotSuggestionsPanelManager.ts new file mode 100644 index 0000000..435f61b --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/copilotSuggestionsPanelManager.ts @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TextDocument, WebviewPanel } from 'vscode'; +import { IVSCodeExtensionContext } from '../../../../../../platform/extContext/common/extensionContext'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { IPosition, ITextDocument } from '../../../lib/src/textDocument'; +import { solutionCountTarget } from '../lib/copilotPanel/common'; +import { BaseSuggestionsPanelManager, ListDocumentInterface } from '../panelShared/baseSuggestionsPanelManager'; +import { PanelCompletion } from './common'; +import { CopilotListDocument } from './copilotListDocument'; +import { CopilotSuggestionsPanel } from './copilotSuggestionsPanel'; +import { copilotPanelConfig } from './panelConfig'; + +export class CopilotSuggestionsPanelManager extends BaseSuggestionsPanelManager { + constructor( + @IInstantiationService instantiationService: IInstantiationService, + @IVSCodeExtensionContext extensionContext: IVSCodeExtensionContext, + ) { + super(copilotPanelConfig, instantiationService, extensionContext); + } + + protected createListDocument( + wrapped: ITextDocument, + position: IPosition, + panel: CopilotSuggestionsPanel + ): ListDocumentInterface { + return this._instantiationService.createInstance(CopilotListDocument, wrapped, position, panel, solutionCountTarget); + } + + protected createSuggestionsPanel( + panel: WebviewPanel, + document: TextDocument, + manager: this + ): CopilotSuggestionsPanel { + return this._instantiationService.createInstance(CopilotSuggestionsPanel, panel, document, manager); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/panelConfig.ts b/completions-sample-code/vscode-node/extension/src/copilotPanel/panelConfig.ts new file mode 100644 index 0000000..256f04c --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/panelConfig.ts @@ -0,0 +1,23 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as constants from '../constants'; +import { CopilotPanelVisible } from '../constants'; +import { PanelConfig } from '../panelShared/basePanelTypes'; + +// Configuration for the GitHub Copilot Suggestions Panel +export const copilotPanelConfig: PanelConfig = { + panelTitle: 'GitHub Copilot Suggestions', + webviewId: 'GitHub Copilot Suggestions', + webviewScriptName: 'suggestionsPanelWebview.js', + contextVariable: CopilotPanelVisible, + commands: { + accept: constants.CMDAcceptCursorPanelSolutionClient, + navigatePrevious: constants.CMDNavigatePreviousPanelSolutionClient, + navigateNext: constants.CMDNavigateNextPanelSolutionClient, + }, + renderingMode: 'streaming', + shuffleSolutions: false, +}; diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/webView/suggestionsPanelWebview.ts b/completions-sample-code/vscode-node/extension/src/copilotPanel/webView/suggestionsPanelWebview.ts new file mode 100644 index 0000000..a2342fa --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/webView/suggestionsPanelWebview.ts @@ -0,0 +1,166 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { provideVSCodeDesignSystem, vsCodeButton } from '@vscode/webview-ui-toolkit'; +import DOMPurify from 'dompurify'; + +const solutionsContainer = document.getElementById('solutionsContainer'); +const vscode = acquireVsCodeApi(); +let currentFocusIndex: number = 0; +let solutionEventHandlersInitialized = false; + +provideVSCodeDesignSystem().register(vsCodeButton()); + +type Message = { + command: string; + solutions: { + htmlSnippet: string; + citation?: { + message: string; + url: string; + }; + }[]; + percentage: number; +}; + +window.addEventListener('DOMContentLoaded', () => { + // Notify the extension that the webview is ready + vscode.postMessage({ command: 'webviewReady' }); + initializeSolutionEventHandlers(); +}); + +window.addEventListener('message', (event) => { + const message = event.data as Message; // The JSON data our extension sent + + switch (message.command) { + case 'solutionsUpdated': + handleSolutionUpdate(message); + break; + case 'navigatePreviousSolution': + navigatePreviousSolution(); + break; + case 'navigateNextSolution': + navigateNextSolution(); + break; + } +}); + +function handleSolutionUpdate(message: Message) { + updateLoadingContainer(message); + + if (solutionsContainer) { + solutionsContainer.innerHTML = message.solutions + .map((solution, index) => { + const renderedCitation = solution.citation + ? `

+ + ${DOMPurify.sanitize(solution.citation.message)} + Inspect source code +

` + : ''; + const sanitizedSnippet = DOMPurify.sanitize(solution.htmlSnippet); + + return `

Suggestion ${index + 1}

+
${sanitizedSnippet + }
+ ${DOMPurify.sanitize(renderedCitation)} + Accept suggestion ${index + 1 + }`; + }) + .join(''); + } +} + +function navigatePreviousSolution() { + const snippets = document.querySelectorAll('.snippetContainer pre'); + const prevIndex = currentFocusIndex - 1; + + snippets[prevIndex]?.focus(); +} + +function navigateNextSolution() { + const snippets = document.querySelectorAll('.snippetContainer pre'); + const nextIndex = (currentFocusIndex ?? -1) + 1; + + if (snippets[nextIndex]) { + snippets[nextIndex].focus(); + } else if (snippets[0]) { + snippets[0].focus(); + } +} + +function updateLoadingContainer(message: Message) { + const progressBar = document.getElementById('progress-bar') as HTMLProgressElement; + const loadingContainer = document.getElementById('loadingContainer') as HTMLDivElement; + if (!progressBar || !loadingContainer) { + return; + } + if (message.percentage >= 100) { + loadingContainer.innerHTML = `${message.solutions.length} Suggestions`; + } else { + const loadingLabelElement = loadingContainer.querySelector('label') as HTMLLabelElement; + if (loadingLabelElement.textContent !== 'Loading suggestions:\u00A0') { + loadingLabelElement.textContent = 'Loading suggestions:\u00A0'; + } + progressBar.value = message.percentage; + } +} + + +function initializeSolutionEventHandlers(): void { + if (solutionEventHandlersInitialized || solutionsContainer === null) { + return; + } + solutionsContainer.addEventListener('focusin', (event) => { + const target = event.target as HTMLElement | null; + const index = extractSolutionIndex(target); + if (index === undefined) { + return; + } + handleFocus(index); + }); + solutionsContainer.addEventListener('click', (event) => { + const target = event.target as HTMLElement | null; + const button = target?.closest('vscode-button[data-solution-index]'); + if (!(button instanceof HTMLElement)) { + return; + } + const index = extractSolutionIndex(button); + if (index === undefined) { + return; + } + handleClick(index); + }); + solutionEventHandlersInitialized = true; +} + +function extractSolutionIndex(element: HTMLElement | null): number | undefined { + const solutionElement = element?.closest('[data-solution-index]'); + if (!(solutionElement instanceof HTMLElement)) { + return undefined; + } + const attributeValue = solutionElement.getAttribute('data-solution-index'); + if (attributeValue === null) { + return undefined; + } + const index = Number.parseInt(attributeValue, 10); + return Number.isNaN(index) ? undefined : index; +} + +function handleFocus(index: number) { + currentFocusIndex = index; + vscode.postMessage({ + command: 'focusSolution', + solutionIndex: index, + }); +} + +function handleClick(index: number) { + vscode.postMessage({ + command: 'acceptSolution', + solutionIndex: index, + }); +} + diff --git a/completions-sample-code/vscode-node/extension/src/copilotPanel/webView/tsconfig.json b/completions-sample-code/vscode-node/extension/src/copilotPanel/webView/tsconfig.json new file mode 100644 index 0000000..574e3a2 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/copilotPanel/webView/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2022", + "skipLibCheck": true, // https://github.com/DataDog/datadog-ci/issues/1059 + "sourceMap": true, + "rootDir": ".", + "lib": ["ES2021", "dom"], + // Reset values set in the parent tsconfig + "strict": true, /* enable all strict type-checking options */ + /* Additional Checks */ + "noUnusedLocals": true, /* Report errors on unused locals. */ + "noImplicitOverride": true, /* Force use of `override` keyword. */ + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "useDefineForClassFields": false, + "resolveJsonModule": true, + "experimentalDecorators": true, + "isolatedModules": false, + }, + "exclude": [], +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/extension/src/extensionStatus.ts b/completions-sample-code/vscode-node/extension/src/extensionStatus.ts new file mode 100644 index 0000000..5bb48ca --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/extensionStatus.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { Command, StatusKind } from '../../types/src'; + +export const ICompletionsExtensionStatus = createServiceIdentifier('ICompletionsExtensionStatus'); +export interface ICompletionsExtensionStatus { + readonly _serviceBrand: undefined; + + kind: StatusKind; + message?: string; + busy: boolean; + command?: Command; +} + +export class CopilotExtensionStatus implements ICompletionsExtensionStatus { + declare _serviceBrand: undefined; + constructor( + public kind: StatusKind = 'Normal', + public message?: string, + public busy = false, + public command?: Command + ) { } +} diff --git a/completions-sample-code/vscode-node/extension/src/fileSystem.ts b/completions-sample-code/vscode-node/extension/src/fileSystem.ts new file mode 100644 index 0000000..c540015 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/fileSystem.ts @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { FileType, Uri, workspace } from 'vscode'; +import { FileIdentifier, FileStat, ICompletionsFileSystemService } from '../../lib/src/fileSystem'; + +class ExtensionFileSystem implements ICompletionsFileSystemService { + declare _serviceBrand: undefined; + + async readFileString(uri: FileIdentifier): Promise { + if (typeof uri !== 'string') { + uri = uri.uri; + } + return new TextDecoder().decode(await workspace.fs.readFile(Uri.parse(uri, true))); + } + async stat(uri: FileIdentifier): Promise { + if (typeof uri !== 'string') { + uri = uri.uri; + } + return await workspace.fs.stat(Uri.parse(uri, true)); + } + async readDirectory(uri: FileIdentifier): Promise<[string, FileType][]> { + if (typeof uri !== 'string') { + uri = uri.uri; + } + return await workspace.fs.readDirectory(Uri.parse(uri, true)); + } +} + +export const extensionFileSystem = new ExtensionFileSystem(); diff --git a/completions-sample-code/vscode-node/extension/src/ghostText/ghostTextProvider.ts b/completions-sample-code/vscode-node/extension/src/ghostText/ghostTextProvider.ts new file mode 100644 index 0000000..4259524 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/ghostText/ghostTextProvider.ts @@ -0,0 +1,127 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + CancellationToken, + InlineCompletionContext, + InlineCompletionEndOfLifeReason, + InlineCompletionEndOfLifeReasonKind, + InlineCompletionItem, + InlineCompletionList, + InlineCompletionTriggerKind, + PartialAcceptInfo, + Position, + Range, + TextDocument, + window +} from 'vscode'; +import { ISurveyService } from '../../../../../../platform/survey/common/surveyService'; +import { assertNever } from '../../../../../../util/vs/base/common/assert'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { createCorrelationId } from '../../../../../inlineEdits/common/correlationId'; +import { CopilotCompletion } from '../../../lib/src/ghostText/copilotCompletion'; +import { handleGhostTextPostInsert, handleGhostTextShown, handlePartialGhostTextPostInsert } from '../../../lib/src/ghostText/last'; +import { GhostText } from '../../../lib/src/inlineCompletion'; +import { telemetry } from '../../../lib/src/telemetry'; +import { wrapDoc } from '../textDocumentManager'; + +export interface GhostTextCompletionList extends InlineCompletionList { + items: GhostTextCompletionItem[]; +} + +export interface GhostTextCompletionItem extends InlineCompletionItem { + copilotCompletion: CopilotCompletion; +} + +export class GhostTextProvider { + + private readonly ghostText: GhostText; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ISurveyService private readonly _surveyService: ISurveyService, + ) { + this.ghostText = this.instantiationService.createInstance(GhostText); + } + + async provideInlineCompletionItems( + vscodeDoc: TextDocument, + position: Position, + context: InlineCompletionContext, + token: CancellationToken + ): Promise { + const textDocument = wrapDoc(vscodeDoc); + if (!textDocument) { + return; + } + + // Opportunity ID is a unique ID generated by the client relating to a single "opportunity" + // to provide some kind of suggestion to the user. Multiple requests might be made for a single + // opportunity, for example requesting a completion as well as an edit suggestion. The single ID + // allows us to correlate the different requests. + const opportunityId = context.requestUuid; + + const formattingOptions = window.visibleTextEditors.find(e => e.document.uri === vscodeDoc.uri)?.options; + + const rawCompletions = await this.ghostText.getInlineCompletions(textDocument, position, token, { + isCycling: context.triggerKind === InlineCompletionTriggerKind.Invoke, + selectedCompletionInfo: context.selectedCompletionInfo, + formattingOptions, + opportunityId, + }); + + if (!rawCompletions) { + return; + } + + const items: GhostTextCompletionItem[] = rawCompletions.map(completion => { + const { start, end } = completion.range; + const newRange = new Range(start.line, start.character, end.line, end.character); + return { + insertText: completion.insertText, + range: newRange, + copilotCompletion: completion, + correlationId: createCorrelationId('completions', {}), + } satisfies GhostTextCompletionItem; + }); + + return { items }; + } + + handleDidShowCompletionItem(item: GhostTextCompletionItem) { + this.instantiationService.invokeFunction(handleGhostTextShown, item.copilotCompletion); + } + + handleDidPartiallyAcceptCompletionItem(item: GhostTextCompletionItem, info: number | PartialAcceptInfo) { + if (typeof info === 'number') { + return; // deprecated API + } + this.instantiationService.invokeFunction(handlePartialGhostTextPostInsert, item.copilotCompletion, info.acceptedLength); + } + + async handleEndOfLifetime(completionItem: GhostTextCompletionItem, reason: InlineCompletionEndOfLifeReason) { + const copilotCompletion = completionItem.copilotCompletion; + switch (reason.kind) { + case InlineCompletionEndOfLifeReasonKind.Accepted: { + this.instantiationService.invokeFunction(handleGhostTextPostInsert, copilotCompletion); + this._surveyService.signalUsage('completions').catch(() => { + // Ignore errors from the survey command execution + }); + return; + } + case InlineCompletionEndOfLifeReasonKind.Rejected: { + this.instantiationService.invokeFunction(telemetry, 'ghostText.dismissed', copilotCompletion.telemetry); + return; + } + case InlineCompletionEndOfLifeReasonKind.Ignored: { + // @ulugbekna: no-op ? + return; + } + default: { + assertNever(reason); + } + } + } +} diff --git a/completions-sample-code/vscode-node/extension/src/icon.ts b/completions-sample-code/vscode-node/extension/src/icon.ts new file mode 100644 index 0000000..37e2809 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/icon.ts @@ -0,0 +1,11 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export enum Icon { + Logo = '$(copilot)', + Warning = '$(copilot-warning)', + NotConnected = '$(copilot-not-connected)', + Blocked = '$(copilot-blocked)', +} diff --git a/completions-sample-code/vscode-node/extension/src/lib/copilotPanel/common.ts b/completions-sample-code/vscode-node/extension/src/lib/copilotPanel/common.ts new file mode 100644 index 0000000..97fded3 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/lib/copilotPanel/common.ts @@ -0,0 +1,6 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export const solutionCountTarget = 10; \ No newline at end of file diff --git a/completions-sample-code/vscode-node/extension/src/lib/copilotPanel/panel.ts b/completions-sample-code/vscode-node/extension/src/lib/copilotPanel/panel.ts new file mode 100644 index 0000000..a221e6c --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/lib/copilotPanel/panel.ts @@ -0,0 +1,137 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IInstantiationService, type ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { asyncIterableMapFilter } from '../../../../lib/src/helpers/iterableHelpers'; +import { ICompletionsLogTargetService, Logger } from '../../../../lib/src/logger'; +import { CopilotUiKind, ICompletionsOpenAIFetcherService } from '../../../../lib/src/openai/fetch'; +import { APIChoice } from '../../../../lib/src/openai/openai'; +import { ICompletionsStatusReporter } from '../../../../lib/src/progress'; +import { getNodeStartUtil } from '../../../../lib/src/prompt/parseBlock'; +import { trimLastLine } from '../../../../lib/src/prompt/prompt'; +import { postProcessChoiceInContext } from '../../../../lib/src/suggestions/suggestions'; +import { LocationFactory } from '../../../../lib/src/textDocument'; +import { + generateSolutionsStream, + reportSolutions, + setupCompletionParams, + setupPromptAndTelemetry, + SolutionManager, + trimChoices, +} from '../panelShared/common'; +import { ISolutionHandler, SolutionsStream, UnformattedSolution } from '../panelShared/panelTypes'; + +const solutionsLogger = new Logger('solutions'); + +/** + * Given an `ISolutionManager` with the context of a specific "Open Copilot" request, + * initiate the generation of a stream of solutions for that request. + */ +export async function launchSolutions(accessor: ServicesAccessor, solutionManager: SolutionManager): Promise { + const instantiationService = accessor.get(IInstantiationService); + const fetcherService = accessor.get(ICompletionsOpenAIFetcherService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const position = solutionManager.targetPosition; + const document = solutionManager.textDocument; + + // Setup prompt and telemetry using shared function + const promptSetup = await setupPromptAndTelemetry(accessor, solutionManager, 'open copilot', solutionsLogger); + if ('status' in promptSetup) { + // This is a SolutionsStream indicating an error occurred + return promptSetup; + } + + const { prompt, trailingWs, telemetryData, repoInfo, ourRequestId } = promptSetup; + + // Setup completion parameters using shared function + const { extra, postOptions, finishedCb, engineInfo } = instantiationService.invokeFunction(setupCompletionParams, + document, + position, + prompt, + solutionManager, + telemetryData + ); + + const cancellationToken = solutionManager.cancellationToken; + + const completionParams = { + prompt, + languageId: document.detectedLanguageId, + repoInfo, + ourRequestId, + engineModelId: engineInfo.modelId, + count: solutionManager.solutionCountTarget, + uiKind: CopilotUiKind.Panel, + postOptions, + headers: engineInfo.headers, + extra, + }; + + const res = await fetcherService.fetchAndStreamCompletions(completionParams, telemetryData.extendedBy(), finishedCb, cancellationToken); + + if (res.type === 'failed' || res.type === 'canceled') { + return { status: 'FinishedWithError', error: `${res.type}: ${res.reason}` }; + } + + let choices: AsyncIterable = res.choices; + choices = trimChoices(choices); + choices = asyncIterableMapFilter(choices, choice => instantiationService.invokeFunction(postProcessChoiceInContext, document, position, choice, false, solutionsLogger)); + + const solutions = asyncIterableMapFilter(choices, async (apiChoice: APIChoice) => { + let display = apiChoice.completionText; + solutionsLogger.info(logTarget, `Open Copilot completion: [${apiChoice.completionText}]`); + + // For completions that can happen in any location in the middle of the code we try to find the existing code + // that should be displayed in the OpenCopilot panel so the code is nicely formatted/highlighted. + // This is not needed for implement unknown function quick fix, as it will be + // always "complete" standalone function in the location suggested by TS' extension. + const displayStartPos = + (await getNodeStartUtil(document, position, apiChoice.completionText)) ?? + LocationFactory.position(position.line, 0); + const [displayBefore] = trimLastLine(document.getText(LocationFactory.range(displayStartPos, position))); + + display = displayBefore + display; + let completionText = apiChoice.completionText; + + if (trailingWs.length > 0 && completionText.startsWith(trailingWs)) { + completionText = completionText.substring(trailingWs.length); + } + + const meanLogProb = apiChoice.meanLogProb; + const meanProb: number = meanLogProb !== undefined ? Math.exp(meanLogProb) : 0; + + const solutionTelemetryData = telemetryData.extendedBy({ + choiceIndex: apiChoice.choiceIndex.toString(), + }); + const solution: UnformattedSolution = { + completionText, + insertText: display, + range: LocationFactory.range(displayStartPos, position), + meanProb: meanProb, + meanLogProb: meanLogProb || 0, + requestId: apiChoice.requestId, + choiceIndex: apiChoice.choiceIndex, + telemetryData: solutionTelemetryData, + copilotAnnotations: apiChoice.copilotAnnotations, + }; + return solution; + }); + // deliberately not awaiting so that we can return quickly + const solutionsStream = generateSolutionsStream(cancellationToken, solutions[Symbol.asyncIterator]()); + return solutionsStream; +} + +export async function runSolutions( + accessor: ServicesAccessor, + solutionManager: SolutionManager, + solutionHandler: ISolutionHandler +): Promise { + const instantiationService = accessor.get(IInstantiationService); + const statusReporter = accessor.get(ICompletionsStatusReporter); + return statusReporter.withProgress(async () => { + const nextSolution = instantiationService.invokeFunction(launchSolutions, solutionManager); + return await reportSolutions(nextSolution, solutionHandler); + }); +} diff --git a/completions-sample-code/vscode-node/extension/src/lib/panelShared/common.ts b/completions-sample-code/vscode-node/extension/src/lib/panelShared/common.ts new file mode 100644 index 0000000..7a01e9d --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/lib/panelShared/common.ts @@ -0,0 +1,299 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken } from 'vscode'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; +import { IInstantiationService, type ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { createCompletionState } from '../../../../lib/src/completionState'; +import { BlockMode } from '../../../../lib/src/config'; +import { ICompletionsFeaturesService } from '../../../../lib/src/experiments/featuresService'; +import { ICompletionsBlockModeConfig } from '../../../../lib/src/ghostText/configBlockMode'; +import { ICompletionsLogTargetService, type Logger } from '../../../../lib/src/logger'; +import { getEngineRequestInfo } from '../../../../lib/src/openai/config'; +import { CompletionHeaders, CompletionRequestExtra, PostOptions } from '../../../../lib/src/openai/fetch'; +import { APIChoice, FinishedCallback } from '../../../../lib/src/openai/openai'; +import { contextIndentation, parsingBlockFinished } from '../../../../lib/src/prompt/parseBlock'; +import { extractPrompt, Prompt } from '../../../../lib/src/prompt/prompt'; +import { extractRepoInfoInBackground, MaybeRepoInfo } from '../../../../lib/src/prompt/repository'; +import { telemetrizePromptLength, telemetry, TelemetryData, TelemetryWithExp } from '../../../../lib/src/telemetry'; +import { IPosition, ITextDocument, LocationFactory, TextDocumentContents } from '../../../../lib/src/textDocument'; +import { isSupportedLanguageId } from '../../../../prompt/src/parse'; +import { Position } from '../../../../types/src'; +import { ISolutionHandler, SolutionsStream, UnformattedSolution } from './panelTypes'; + +export const solutionCountTarget = 10; + +export function panelPositionForDocument(document: TextDocumentContents, position: Position): IPosition { + let returnPosition = position; + const line = document.lineAt(position.line); + if (!line.isEmptyOrWhitespace) { + returnPosition = line.range.end; + } + return returnPosition; +} + +/** + * Trim trailing whitespace. + */ +export async function* trimChoices(choices: AsyncIterable): AsyncIterable { + for await (const choice of choices) { + const choiceCopy = { ...choice }; + choiceCopy.completionText = choiceCopy.completionText.trimEnd(); + yield choiceCopy; + } +} + +export class SolutionManager { + private _savedTelemetryData?: TelemetryWithExp | undefined; + readonly targetPosition = panelPositionForDocument(this.textDocument, this.startPosition); + + constructor( + readonly textDocument: ITextDocument, + public startPosition: IPosition, + readonly cancellationToken: CancellationToken, + readonly solutionCountTarget: number + ) { } + + get savedTelemetryData(): TelemetryWithExp | undefined { + return this._savedTelemetryData; + } + + set savedTelemetryData(data: TelemetryWithExp | undefined) { + this._savedTelemetryData = data; + } +} + +export async function reportSolutions( + nextSolutionPromise: Promise, + solutionHandler: ISolutionHandler +): Promise { + const nextSolution = await nextSolutionPromise; + switch (nextSolution.status) { + case 'Solution': + await solutionHandler.onSolution(nextSolution.solution); + await reportSolutions(nextSolution.next, solutionHandler); + break; + case 'FinishedNormally': + await solutionHandler.onFinishedNormally(); + break; + case 'FinishedWithError': + await solutionHandler.onFinishedWithError(nextSolution.error); + break; + } +} + +export async function generateSolutionsStream( + cancellationToken: CancellationToken, + solutions: AsyncIterator +): Promise { + if (cancellationToken.isCancellationRequested) { + return { status: 'FinishedWithError', error: 'Cancelled' }; + } + const nextResult = await solutions.next(); + if (nextResult.done === true) { + return { status: 'FinishedNormally' }; + } + return { + status: 'Solution', + solution: nextResult.value, + next: generateSolutionsStream(cancellationToken, solutions), + }; +} + +export function normalizeCompletionText(text: string): string { + return text.replace(/\s+/g, ''); +} + +/** + * Result of prompt processing setup + */ +export interface PromptSetupResult { + prompt: Prompt; + trailingWs: string; + telemetryData: TelemetryWithExp; + repoInfo: MaybeRepoInfo; + ourRequestId: string; +} + +/** + * Sets up prompt extraction, telemetry, and handles common error cases. + * Returns null if an error occurred that should terminate processing. + */ +export async function setupPromptAndTelemetry( + accessor: ServicesAccessor, + solutionManager: SolutionManager, + source: 'open copilot' | 'open comparison', + solutionsLogger: Logger, + engineName?: string, + comparisonRequestId?: string +): Promise { + const position = solutionManager.targetPosition; + const document = solutionManager.textDocument; + + const repoInfo = extractRepoInfoInBackground(accessor, document.uri); + + // Telemetry setup + const ourRequestId = generateUuid(); + const tempTelemetry = TelemetryData.createAndMarkAsIssued( + { + headerRequestId: ourRequestId, + languageId: document.detectedLanguageId, + source, + }, + {} + ); + + const featuresService = accessor.get(ICompletionsFeaturesService); + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + // Update telemetry with experiment values + solutionManager.savedTelemetryData = await featuresService + .fetchTokenAndUpdateExPValuesAndAssignments( + { uri: document.uri, languageId: document.detectedLanguageId }, + tempTelemetry + ); + + // Add in comparison panel specific info + if (engineName) { + solutionManager.savedTelemetryData = solutionManager.savedTelemetryData!.extendedBy({ + engineName, + }); + } + if (comparisonRequestId) { + solutionManager.savedTelemetryData = solutionManager.savedTelemetryData!.extendedBy({ + comparisonRequestId, + }); + } + + // Extract prompt + const promptResponse = await instantiationService.invokeFunction(extractPrompt, + ourRequestId, + createCompletionState(document, position), + solutionManager.savedTelemetryData! + ); + + // Handle prompt extraction errors + if (promptResponse.type === 'copilotContentExclusion') { + return { status: 'FinishedNormally' }; + } + if (promptResponse.type === 'contextTooShort') { + return { status: 'FinishedWithError', error: 'Context too short' }; + } + if (promptResponse.type === 'promptCancelled') { + return { status: 'FinishedWithError', error: 'Prompt cancelled' }; + } + if (promptResponse.type === 'promptTimeout') { + return { status: 'FinishedWithError', error: 'Prompt timeout' }; + } + if (promptResponse.type === 'promptError') { + return { status: 'FinishedWithError', error: 'Prompt error' }; + } + + const prompt = promptResponse.prompt; + const trailingWs = promptResponse.trailingWs; + + // Handle trailing whitespace adjustment + if (trailingWs.length > 0) { + solutionManager.startPosition = LocationFactory.position( + solutionManager.startPosition.line, + solutionManager.startPosition.character - trailingWs.length + ); + } + + // Update telemetry with prompt information + solutionManager.savedTelemetryData = solutionManager.savedTelemetryData!.extendedBy( + {}, + { + ...telemetrizePromptLength(prompt), + solutionCount: solutionManager.solutionCountTarget, + promptEndPos: document.offsetAt(position), + } + ); + + solutionsLogger.debug(logTarget, 'prompt:', prompt); + instantiationService.invokeFunction(telemetry, 'solution.requested', solutionManager.savedTelemetryData); + + return { + prompt, + trailingWs, + telemetryData: solutionManager.savedTelemetryData, + repoInfo, + ourRequestId, + }; +} + +/** + * Result of completion parameters setup + */ +export interface CompletionSetupResult { + extra: CompletionRequestExtra; + postOptions: PostOptions; + finishedCb: FinishedCallback; + engineInfo: { modelId: string; headers: CompletionHeaders }; +} + +/** + * Sets up block mode, completion parameters, and finished callback. + */ +export function setupCompletionParams( + accessor: ServicesAccessor, + document: ITextDocument, + position: IPosition, + prompt: Prompt, + solutionManager: SolutionManager, + telemetryData: TelemetryWithExp +): CompletionSetupResult { + // Compute block mode + const blockMode = accessor.get(ICompletionsBlockModeConfig).forLanguage(document.detectedLanguageId, telemetryData); + const isSupportedLanguage = isSupportedLanguageId(document.detectedLanguageId); + + const contextIndent = contextIndentation(document, position); + const extra: CompletionRequestExtra = { + language: document.detectedLanguageId, + next_indent: contextIndent.next ?? 0, + prompt_tokens: prompt.prefixTokens ?? 0, + suffix_tokens: prompt.suffixTokens ?? 0, + }; + + const postOptions: PostOptions = {}; + if (blockMode === BlockMode.Parsing && !isSupportedLanguage) { + postOptions['stop'] = ['\n\n', '\r\n\r\n']; + } + + const engineInfo = getEngineRequestInfo(accessor, telemetryData); + + let finishedCb: FinishedCallback; + + switch (blockMode) { + case BlockMode.Server: + // Client knows the block is done when the completion is. + finishedCb = () => undefined; + // If requested at the top-level, don't trim at all. + extra.force_indent = contextIndent.prev ?? -1; + extra.trim_by_indentation = true; + break; + case BlockMode.ParsingAndServer: + finishedCb = isSupportedLanguage + ? parsingBlockFinished(document, solutionManager.startPosition) + : () => undefined; + // If requested at the top-level, don't trim at all. + extra.force_indent = contextIndent.prev ?? -1; + extra.trim_by_indentation = true; + break; + case BlockMode.Parsing: + default: + finishedCb = isSupportedLanguage + ? parsingBlockFinished(document, solutionManager.startPosition) + : () => undefined; + break; + } + + return { + extra, + postOptions, + finishedCb, + engineInfo, + }; +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/extension/src/lib/panelShared/panelTypes.ts b/completions-sample-code/vscode-node/extension/src/lib/panelShared/panelTypes.ts new file mode 100644 index 0000000..4d1e382 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/lib/panelShared/panelTypes.ts @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotNamedAnnotationList } from '../../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { RequestId } from '../../../../../../../platform/networking/common/fetch'; +import { TelemetryWithExp } from '../../../../lib/src/telemetry'; +import { IRange } from '../../../../lib/src/textDocument'; + +export interface UnformattedSolution { + /** Raw text returned by model */ + completionText: string; + /** Text that should be inserted into the document, replacing the text at .range */ + insertText: string; + range: IRange; + meanProb: number; + meanLogProb: number; + requestId: RequestId; + choiceIndex: number; + telemetryData: TelemetryWithExp; + copilotAnnotations?: CopilotNamedAnnotationList; + /** Optional Model ID when fetching from multiple models */ + modelId?: string; +} + +export interface ISolutionHandler { + onSolution(solution: UnformattedSolution): Promise | void; + onFinishedNormally(): Promise | void; + onFinishedWithError(error: string): Promise | void; +} + +/** + * A stream of solutions, ending either with 'FinishedNormally' or 'FinishedWithError'. + * This structure allows for errors to occur part way through the stream, as well as + * at the beginning. + * + * The stream is similar to an async generator, but with more information when the stream + * ends: instead of just `done` we can have `FinishedNormally` or `FinishedWithError`. + */ +export type SolutionsStream = + | { status: 'FinishedNormally' } + | { status: 'FinishedWithError'; error: string } + | { status: 'Solution'; solution: UnformattedSolution; next: Promise }; diff --git a/completions-sample-code/vscode-node/extension/src/modelPicker.ts b/completions-sample-code/vscode-node/extension/src/modelPicker.ts new file mode 100644 index 0000000..77af334 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/modelPicker.ts @@ -0,0 +1,148 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { env, QuickPick, QuickPickItem, QuickPickItemKind, Uri, window, workspace } from 'vscode'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ConfigKey, getConfig } from '../../lib/src/config'; +import { CopilotConfigPrefix } from '../../lib/src/constants'; +import { AsyncCompletionManager, ICompletionsAsyncManagerService } from '../../lib/src/ghostText/asyncCompletions'; +import { CompletionsCache, ICompletionsCacheService } from '../../lib/src/ghostText/completionsCache'; +import { ICompletionsLogTargetService, Logger } from '../../lib/src/logger'; +import { AvailableModelsManager, ICompletionsModelManagerService, ModelItem } from '../../lib/src/openai/model'; +import { telemetry, TelemetryData } from '../../lib/src/telemetry'; +const logger = new Logger('modelPicker'); + +interface ModelPickerItem extends Omit, QuickPickItem { + // Distinguish between items in the quick pick + type: 'model' | 'separator' | 'learn-more'; +} + +// Separator and learn-more links are always shown in the quick pick +const defaultModelPickerItems: ModelPickerItem[] = [ + // Add separator after the models + { + label: '', + kind: QuickPickItemKind.Separator, + modelId: 'separator', + type: 'separator' as const, + alwaysShow: true, + }, + // Add "Learn more" item at the end + { + modelId: 'learn-more', + label: 'Learn more $(link-external)', + description: '', + alwaysShow: true, + type: 'learn-more' as const, + }, +]; + +export class ModelPickerManager { + // URL for information about Copilot models + private readonly MODELS_INFO_URL = 'https://aka.ms/CopilotCompletionsModelPickerLearnMore'; + + get models(): ModelItem[] { + return this._modelManager.getGenericCompletionModels(); + } + + private getDefaultModelId(): string { + return this._modelManager.getDefaultModelId(); + } + + constructor( + @IInstantiationService private readonly _instantiationService: IInstantiationService, + @ICompletionsAsyncManagerService private readonly _asyncCompletionManager: AsyncCompletionManager, + @ICompletionsModelManagerService private readonly _modelManager: AvailableModelsManager, + @ICompletionsLogTargetService private readonly _logTarget: ICompletionsLogTargetService, + @ICompletionsCacheService private readonly _completionsCache: CompletionsCache + ) { } + + async setUserSelectedCompletionModel(modelId: string | null) { + return workspace + .getConfiguration(CopilotConfigPrefix) + .update(ConfigKey.UserSelectedCompletionModel, modelId ?? '', true); + } + + async handleModelSelection(quickpickList: QuickPick) { + const model = quickpickList.activeItems[0]; + if (model === undefined) { + return; + } + quickpickList.hide(); + + // Open up the link + if (model.type === 'learn-more') { + await env.openExternal(Uri.parse(this.MODELS_INFO_URL)); + this._instantiationService.invokeFunction(telemetry, 'modelPicker.learnMoreClicked'); + return; + } + + await this.selectModel(model); + } + + async selectModel(model: ModelPickerItem) { + const currentModel = this._instantiationService.invokeFunction(getUserSelectedModelConfiguration); + + if (currentModel !== model.modelId) { + this._completionsCache.clear(); + this._asyncCompletionManager.clear(); + } + + const modelSelection = model.modelId === this.getDefaultModelId() ? null : model.modelId; + await this.setUserSelectedCompletionModel(modelSelection); + if (modelSelection === null) { + logger.info(this._logTarget, `User selected default model; setting null`); + } else { + logger.info(this._logTarget, `Selected model: ${model.modelId}`); + } + + this._instantiationService.invokeFunction( + telemetry, + 'modelPicker.modelSelected', + TelemetryData.createAndMarkAsIssued({ + engineName: modelSelection ?? 'default', + }) + ); + } + + private modelsForModelPicker(): [string | null, ModelPickerItem[]] { + const currentModelSelection = this._instantiationService.invokeFunction(getUserSelectedModelConfiguration); + const items: ModelPickerItem[] = this.models.map(model => { + return { + modelId: model.modelId, + label: `${model.label}${model.preview ? ' (Preview)' : ''}`, + description: `(${model.modelId})`, + alwaysShow: model.modelId === this.getDefaultModelId(), + type: 'model' as const, + }; + }); + + return [currentModelSelection, items]; + } + + showModelPicker(): QuickPick { + const [currentModelSelection, items] = this.modelsForModelPicker(); + + const quickPick = window.createQuickPick(); + quickPick.title = 'Change Completions Model'; + quickPick.items = [...items, ...defaultModelPickerItems]; + quickPick.onDidAccept(() => this.handleModelSelection(quickPick)); + + const currentModelOrDefault = currentModelSelection ?? this.getDefaultModelId(); + + // set the currently selected model as active + const selectedItem = quickPick.items.find(item => item.modelId === currentModelOrDefault); + if (selectedItem) { + quickPick.activeItems = [selectedItem]; + } + + quickPick.show(); + return quickPick; + } +} + +function getUserSelectedModelConfiguration(accessor: ServicesAccessor): string | null { + const value = getConfig(accessor, ConfigKey.UserSelectedCompletionModel); + return typeof value === 'string' && value.length > 0 ? value : null; +} diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/baseListDocument.ts b/completions-sample-code/vscode-node/extension/src/panelShared/baseListDocument.ts new file mode 100644 index 0000000..ab2ec20 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/baseListDocument.ts @@ -0,0 +1,110 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Position, Range } from 'vscode'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { postInsertionTasks } from '../../../lib/src/postInsertion'; +import { countLines } from '../../../lib/src/suggestions/partialSuggestions'; +import { IPosition, ITextDocument } from '../../../lib/src/textDocument'; +import { normalizeCompletionText, solutionCountTarget, SolutionManager } from '../lib/panelShared/common'; +import { UnformattedSolution } from '../lib/panelShared/panelTypes'; +import { BasePanelCompletion, ISuggestionsPanel } from './basePanelTypes'; + +// BaseListDocument to be shared with both the copilot and comparison completion panels. +export abstract class BaseListDocument extends SolutionManager { + private _solutionCount = 0; + protected readonly _solutions: TPanelCompletion[] = []; + + constructor( + textDocument: ITextDocument, + position: IPosition, + readonly panel: ISuggestionsPanel, + countTarget = solutionCountTarget, + @IInstantiationService protected readonly instantiationService: IInstantiationService + ) { + super(textDocument, position, panel.cancellationToken, countTarget); + } + + protected abstract createPanelCompletion( + unformatted: UnformattedSolution, + baseCompletion: BasePanelCompletion + ): TPanelCompletion; + protected abstract shouldAddSolution(newItem: TPanelCompletion): boolean; + protected abstract runSolutionsImpl(): Promise; + + // Find if two solutions are duplicates by comparing their normalized text content. + protected areSolutionsDuplicates(solutionA: TPanelCompletion, solutionB: TPanelCompletion): boolean { + const stripA = normalizeCompletionText(solutionA.insertText); + const stripB = normalizeCompletionText(solutionB.insertText); + return stripA === stripB; + } + + protected findDuplicateSolution(newItem: TPanelCompletion): TPanelCompletion | undefined { + return this._solutions.find(item => this.areSolutionsDuplicates(item, newItem)); + } + + onSolution(unformatted: UnformattedSolution) { + const offset = this.textDocument.offsetAt(this.targetPosition); + const rank = this._solutions.length; + + const postInsertionCallback = () => { + const telemetryData = this.savedTelemetryData!.extendedBy( + { + choiceIndex: unformatted.choiceIndex.toString(), + engineName: unformatted.modelId || '', + }, + { + compCharLen: unformatted.insertText.length, + meanProb: unformatted.meanProb, + rank, + } + ); + return this.instantiationService.invokeFunction(postInsertionTasks, + 'solution', + unformatted.insertText, + offset, + this.textDocument.uri, + telemetryData, + { + compType: 'full', + acceptedLength: unformatted.insertText.length, + acceptedLines: countLines(unformatted.insertText), + }, + unformatted.copilotAnnotations + ); + }; + + const baseCompletion: BasePanelCompletion = { + insertText: unformatted.insertText, + range: new Range( + new Position(unformatted.range.start.line, unformatted.range.start.character), + new Position(unformatted.range.end.line, unformatted.range.end.character) + ), + copilotAnnotations: unformatted.copilotAnnotations, + postInsertionCallback, + }; + + const newItem = this.createPanelCompletion(unformatted, baseCompletion); + + if (this.shouldAddSolution(newItem)) { + this.panel.onItem(newItem); + this._solutions.push(newItem); + } + this._solutionCount++; + this.panel.onWorkDone({ percentage: (100 * this._solutionCount) / this.solutionCountTarget }); + } + + onFinishedNormally() { + return this.panel.onFinished(); + } + + onFinishedWithError(_: string) { + return this.onFinishedNormally(); + } + + runQuery() { + return this.runSolutionsImpl(); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/basePanelTypes.ts b/completions-sample-code/vscode-node/extension/src/panelShared/basePanelTypes.ts new file mode 100644 index 0000000..a6d89fd --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/basePanelTypes.ts @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken, Range } from 'vscode'; +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; + +// Base interface for a completion displayed in the panel. +export interface BasePanelCompletion { + insertText: string; + range: Range; + copilotAnnotations?: CopilotNamedAnnotationList; + postInsertionCallback: () => PromiseLike | void; +} + +// Interface for the suggestions panel, which handles work done notifications and item selections. +export interface ISuggestionsPanel { + cancellationToken: CancellationToken; + onWorkDone(_: { percentage: number }): void; + onItem(_: BasePanelCompletion): void; + onFinished(): void; +} + +// Configuration for webview panels for completions. +export interface PanelConfig { + panelTitle: string; + webviewId: string; + webviewScriptName: string; + contextVariable: string; + commands: { + accept: string; + navigatePrevious: string; + navigateNext: string; + }; + renderingMode: 'streaming' | 'batch'; + shuffleSolutions: boolean; +} + +// Configuration for webview panels, used to pass settings to the webview. +export interface WebviewConfig { + renderingMode: 'batch' | 'streaming'; + shuffleSolutions: boolean; +} diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/baseSuggestionsPanel.ts b/completions-sample-code/vscode-node/extension/src/panelShared/baseSuggestionsPanel.ts new file mode 100644 index 0000000..5cdea50 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/baseSuggestionsPanel.ts @@ -0,0 +1,338 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + CancellationTokenSource, + Disposable, + Event, + EventEmitter, + TextDocument, + Uri, + WebviewPanel, + WorkspaceEdit, + commands, + workspace, +} from 'vscode'; +import { IVSCodeExtensionContext } from '../../../../../../platform/extContext/common/extensionContext'; +import { debounce } from '../../../../../../util/common/debounce'; +import { BasePanelCompletion, ISuggestionsPanel, PanelConfig } from './basePanelTypes'; +import { Highlighter } from './highlighter'; +import { getNonce, pluralize } from './utils'; + +//import { IPCitationDetail } from '#lib/citationManager'; +interface IPCitationDetail { + license: string; + url: string; +} + +export interface SuggestionsPanelManagerInterface { + activeWebviewPanel: BaseSuggestionsPanel | undefined; + decrementPanelCount(): void; +} + +export interface SolutionContent { + htmlSnippet: string; + citation?: { message: string; url: string }; + [key: string]: unknown; // Allow additional properties for panel-specific content +} + +export interface BaseWebviewMessage { + command: string; +} + +interface AcceptSolutionMessage extends BaseWebviewMessage { + command: 'acceptSolution'; + solutionIndex: number; +} + +interface FocusSolutionMessage extends BaseWebviewMessage { + command: 'focusSolution'; + solutionIndex: number; +} + +interface SubmitFeedbackMessage extends BaseWebviewMessage { + command: 'submitFeedback'; + solutionIndex: number; + feedback: string; +} + +interface RefreshMessage extends BaseWebviewMessage { + command: 'refresh'; +} + +interface WebviewReadyMessage extends BaseWebviewMessage { + command: 'webviewReady'; +} + +export type WebviewMessage = + | AcceptSolutionMessage + | FocusSolutionMessage + | SubmitFeedbackMessage + | RefreshMessage + | WebviewReadyMessage; + +export abstract class BaseSuggestionsPanel implements ISuggestionsPanel { + private _disposables: Disposable[] = []; + #items: TPanelCompletion[] = []; + #batchItems: TPanelCompletion[] = []; + #percentage = 0; + #highlighter: Thenable; + private _focusedSolution: TPanelCompletion | undefined; + private _isDisposed: boolean = false; + #documentUri: Uri; + #cts = new CancellationTokenSource(); + + private _onDidDispose = new EventEmitter(); + readonly onDidDispose: Event = this._onDidDispose.event; + + get cancellationToken() { + return this.#cts.token; + } + + constructor( + readonly webviewPanel: WebviewPanel, + document: TextDocument, + protected suggestionsPanelManager: SuggestionsPanelManagerInterface, + protected readonly config: PanelConfig, + @IVSCodeExtensionContext protected readonly contextService: IVSCodeExtensionContext, + ) { + webviewPanel.onDidDispose(() => this._dispose(), null, this._disposables); + webviewPanel.webview.html = this._getWebviewContent(); + this.#documentUri = document.uri; + + this.#highlighter = Highlighter.create(document.languageId); + + workspace.onDidChangeConfiguration(e => { + if (e.affectsConfiguration('workbench.colorTheme')) { + return this.render(); + } + }); + + webviewPanel.webview.onDidReceiveMessage(async (message: WebviewMessage) => { + // First lest the subclass handle custom messages + if ((await this.handleCustomMessage(message)) === true) { + return; + } + switch (message.command) { + case 'focusSolution': + this._focusedSolution = this.#items[message.solutionIndex]; + return; + case 'webviewReady': + // Send the config to the webview + void this.postMessage({ + command: 'updateConfig', + config: { + renderingMode: this.config.renderingMode, + shuffleSolutions: this.config.shuffleSolutions, + }, + }); + return; + } + }, undefined); + + webviewPanel.onDidChangeViewState(e => { + if (e.webviewPanel?.visible) { + this.suggestionsPanelManager.activeWebviewPanel = this; + } + }); + } + + protected async handleCustomMessage(message: BaseWebviewMessage): Promise { + return Promise.resolve(false); + } + protected abstract renderSolutionContent(item: TPanelCompletion, baseContent: SolutionContent): SolutionContent; + + private _buildExtensionUri(...path: string[]): Uri { + const extensionPath = Uri.joinPath(this.contextService.extensionUri, ...path); + return this.webviewPanel.webview.asWebviewUri(extensionPath); + } + + private _getWebviewContent() { + const nonce = getNonce(); + const scriptUri = this._buildExtensionUri('dist', this.config.webviewScriptName); + + return ` + + + + + + + ${this.config.panelTitle} + + + +

${this.config.panelTitle}

+
+ + +
+
+ + + + `; + } + + onWorkDone({ percentage }: { percentage: number }) { + this.#percentage = percentage; + void this.render(); + } + + onItem(item: TPanelCompletion) { + // If rendering mode is 'batch', we collect items and render them later + // Otherwise, we render immediately + if (this.config.renderingMode === 'batch') { + this.#batchItems.push(item); + } else { + this.#items.push(item); + void this.render(); + } + } + + clearSolutions() { + // Cancel any ongoing operations + this.#cts.cancel(); + // Create a new cancellation token source for the next operation + this.#cts = new CancellationTokenSource(); + + // Clear all solutions and reset state + this.#items = []; + this.#batchItems = []; + this._focusedSolution = undefined; + this.#percentage = 0; + void this.render(); + } + + onFinished() { + this.#percentage = 100; + + // If we have batch items, add them to the main items list, shuffle if needed, and render + if (this.#batchItems.length > 0) { + this.#items.push(...this.#batchItems); + + if (this.config.shuffleSolutions) { + this.#items = this.#items.sort(() => Math.random() - 0.5); + } + + this.#batchItems = []; + } + + void this.render(); + } + + protected async acceptSolution(solution: TPanelCompletion, closePanel: boolean = true) { + if (this._isDisposed === false && solution?.range) { + const edit = new WorkspaceEdit(); + edit.replace(this.#documentUri, solution.range, solution.insertText); + await workspace.applyEdit(edit); + this.#cts.cancel(); + if (closePanel) { + await commands.executeCommand('workbench.action.closeActiveEditor'); + } + await solution.postInsertionCallback(); + } + } + + protected items(): TPanelCompletion[] { + return this.#items; + } + + async acceptFocusedSolution() { + const solution = this._focusedSolution; + if (solution) { + return this.acceptSolution(solution); + } + } + + protected async renderSolutions() { + const highlighter = await this.#highlighter; + const content = this.#items.map(item => { + const firstCitation = item.copilotAnnotations?.ip_code_citations?.[0]; + const details = firstCitation?.details.citations as IPCitationDetail[] | undefined; + let renderedCitatation: { message: string; url: string } | undefined; + if (details && details.length > 0) { + const licensesSet = new Set(details.map(d => d.license)); + if (licensesSet.has('NOASSERTION')) { + licensesSet.delete('NOASSERTION'); + licensesSet.add('unknown'); + } + const allLicenses = Array.from(licensesSet).sort(); + const licenseString = allLicenses.length === 1 ? allLicenses[0] : `[${allLicenses.join(', ')}]`; + renderedCitatation = { + message: `Similar code with ${pluralize(allLicenses.length, 'license type')} ${licenseString} detected.`, + url: details[0].url, + }; + } + + const baseContent = { + htmlSnippet: highlighter.createSnippet(item.insertText.trim()), + citation: renderedCitatation, + }; + + return this.renderSolutionContent(item, baseContent); + }); + + const message = this.createSolutionsMessage(content, this.#percentage); + await this.postMessage(message); + } + + // Subclasses must implement this to create their specific message format + protected abstract createSolutionsMessage(content: SolutionContent[], percentage: number): unknown; + + render = debounce(10, () => this.renderSolutions()); + + postMessage(message: unknown) { + if (this._isDisposed === false) { + return this.webviewPanel.webview.postMessage(message); + } + } + + private _dispose() { + this._isDisposed = true; + this._onDidDispose.fire(); + this.suggestionsPanelManager.decrementPanelCount(); + while (this._disposables.length) { + const disposable = this._disposables.pop(); + if (disposable) { + disposable.dispose(); + } + } + this._onDidDispose.dispose(); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/baseSuggestionsPanelManager.ts b/completions-sample-code/vscode-node/extension/src/panelShared/baseSuggestionsPanelManager.ts new file mode 100644 index 0000000..eafba6e --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/baseSuggestionsPanelManager.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TextDocument, Uri, ViewColumn, WebviewPanel, commands, window } from 'vscode'; +import { IVSCodeExtensionContext } from '../../../../../../platform/extContext/common/extensionContext'; +import { DisposableStore, IDisposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { IPosition, ITextDocument } from '../../../lib/src/textDocument'; +import { basename } from '../../../lib/src/util/uri'; +import { registerCommandWrapper } from '../telemetry'; +import { BasePanelCompletion, PanelConfig } from './basePanelTypes'; +import { BaseSuggestionsPanel, SuggestionsPanelManagerInterface } from './baseSuggestionsPanel'; + +export interface ListDocumentInterface { + runQuery(): Promise; +} + +export abstract class BaseSuggestionsPanelManager + implements SuggestionsPanelManagerInterface { + activeWebviewPanel: BaseSuggestionsPanel | undefined; + private _panelCount: number = 0; + + constructor( + protected readonly config: PanelConfig, + @IInstantiationService protected readonly _instantiationService: IInstantiationService, + @IVSCodeExtensionContext protected readonly _extensionContext: IVSCodeExtensionContext, + ) { } + + protected abstract createListDocument( + wrapped: ITextDocument, + position: IPosition, + panel: BaseSuggestionsPanel + ): ListDocumentInterface; + + protected abstract createSuggestionsPanel( + panel: WebviewPanel, + document: TextDocument, + manager: this + ): BaseSuggestionsPanel; + + renderPanel( + document: TextDocument, + position: IPosition, + wrapped: ITextDocument + ): BaseSuggestionsPanel { + const title = `${this.config.panelTitle} for ${basename(document.uri.toString()) || document.uri.toString()}`; + const panel = window.createWebviewPanel(this.config.webviewId, title, ViewColumn.Two, { + enableScripts: true, + localResourceRoots: [Uri.joinPath(this._extensionContext.extensionUri, 'dist')], + retainContextWhenHidden: true, + }); + + const suggestionPanel = this.createSuggestionsPanel(panel, document, this); + // Listen for the panel disposal event to clear our reference + suggestionPanel.onDidDispose(() => { + if (this.activeWebviewPanel === suggestionPanel) { + this.activeWebviewPanel = undefined; + } + }); + + void this.createListDocument(wrapped, position, suggestionPanel).runQuery(); + + this.activeWebviewPanel = suggestionPanel; + this._panelCount = this._panelCount + 1; + return suggestionPanel; + } + + registerCommands(): IDisposable { + const disposableStore = new DisposableStore(); + + disposableStore.add(this._instantiationService.invokeFunction(registerCommandWrapper, this.config.commands.accept, () => { + return this.activeWebviewPanel?.acceptFocusedSolution(); + })); + + disposableStore.add(this._instantiationService.invokeFunction(registerCommandWrapper, this.config.commands.navigatePrevious, () => { + return this.activeWebviewPanel?.postMessage({ + command: 'navigatePreviousSolution', + }); + })); + + disposableStore.add(this._instantiationService.invokeFunction(registerCommandWrapper, this.config.commands.navigateNext, () => { + return this.activeWebviewPanel?.postMessage({ + command: 'navigateNextSolution', + }); + })); + + return disposableStore; + } + + decrementPanelCount() { + this._panelCount = this._panelCount - 1; + if (this._panelCount === 0) { + void commands.executeCommand('setContext', this.config.contextVariable, false); + } + } +} diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/highlighter.ts b/completions-sample-code/vscode-node/extension/src/panelShared/highlighter.ts new file mode 100644 index 0000000..1d658ec --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/highlighter.ts @@ -0,0 +1,117 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { getSingletonHighlighterCore, HighlighterCore, ThemeRegistration, ThemeRegistrationAny } from 'shiki/core'; +import * as langs from 'shiki/langs'; +import { BundledLanguage } from 'shiki/langs'; +import getWasmInlined from 'shiki/wasm'; +import { ColorThemeKind, window, workspace } from 'vscode'; +import * as languages from './languages'; +import * as themes from './themes'; + +export class Highlighter { + private constructor( + private languageId: string | undefined, + private highlighter: HighlighterCore | undefined + ) { } + + static async create(languageId = window.activeTextEditor?.document.languageId): Promise { + if (!languageId) { + return new Highlighter(undefined, undefined); + } + + const highlighter = await getSingletonHighlighterCore({ + langs: Object.values(langs.bundledLanguages), + loadWasm: getWasmInlined, + }); + + // Load additional language if not out of the box for shiki + if (!langs.bundledLanguages[languageId as BundledLanguage]) { + const additionalLang = vscLanguageMap[languageId as keyof typeof vscLanguageMap]; + if (additionalLang) { + await highlighter.loadLanguage(additionalLang); + } + } + + return new Highlighter(languageId, highlighter); + } + + createSnippet(text: string): string { + if (!this.highlighter || !this.languageId || !this.languageSupported()) { + return `
${text}
`; + } + + return this.highlighter.codeToHtml(text, { lang: this.languageId, theme: getCurrentTheme() }); + } + + private languageSupported() { + if (!this.languageId) { return false; } + + if (this.highlighter?.getLoadedLanguages().includes(this.languageId)) { + return true; + } + + return false; + } +} + +function getCurrentTheme(): ThemeRegistration { + const workbenchConfig = workspace.getConfiguration('workbench'); + if (workbenchConfig) { + const vsCodeTheme = workbenchConfig.get('colorTheme'); + if (vsCodeTheme && isSupportedTheme(vsCodeTheme)) { + return vscThemeMap[vsCodeTheme]; + } + const themeType = window.activeColorTheme; + const defaultTheme = vscDefaultMap[themeType.kind]; // fall back to default themes if we don't have a match + + return defaultTheme; + } else { + return vscThemeMap['Default Dark Modern']; + } +} + +const vscDefaultMap: { [key in ColorThemeKind]: ThemeRegistrationAny } = { + [ColorThemeKind.Dark]: themes.darkModern, + [ColorThemeKind.Light]: themes.lightModern, + [ColorThemeKind.HighContrast]: themes.darkHC, + [ColorThemeKind.HighContrastLight]: themes.lightHC, +}; + +// These are vs code themes that aren't out of the box in shiki but come standard with vs code +const vscThemeMap: { [key: string]: ThemeRegistrationAny } = { + Abyss: themes.abyss, + 'Dark High Contrast': themes.darkHC, + 'Light High Constrast': themes.lightHC, + 'Default Dark Modern': themes.darkModern, + 'Kimbie Dark': themes.kimbieDark, + 'Default Light Modern': themes.lightModern, + 'Monokai Dimmed': themes.monokaiDim, + 'Quiet Light': themes.quietLight, + Red: themes.red, + 'Tomorrow Night Blue': themes.tomorrowNightBlue, + 'Visual Studio Dark': themes.vsDark, + 'Visual Studio Light': themes.vsLight, + 'Default Dark+': themes.darkPlus, + 'Default Light+': themes.lightPlus, + Monokai: themes.monokai, + 'Solarized Dark': themes.solarizedDark, + 'Solarized Light': themes.solarizedLight, +} as const; + +function isSupportedTheme(theme: keyof typeof vscThemeMap): theme is keyof typeof vscThemeMap { + return theme in vscThemeMap; +} + +// These are vs code themes that aren't out of the box in shiki but come standard with vs code +const vscLanguageMap = { + 'cuda-cpp': languages.cudaCpp, + javascriptreact: languages.javascriptreact, + markdown_latex_combined: languages.markdownLatexCombined, + 'markdown-math': languages.markdownMath, + restructuredtext: languages.restructuredtext, + 'search-result': languages.searchResult, + typescriptreact: languages.typescriptreact, +} as const; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/languages/cuda-cpp.tmLanguage.ts b/completions-sample-code/vscode-node/extension/src/panelShared/languages/cuda-cpp.tmLanguage.ts new file mode 100644 index 0000000..4c7dc37 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/languages/cuda-cpp.tmLanguage.ts @@ -0,0 +1,19825 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/* eslint-disable local/no-unexternalized-strings */ +import { LanguageInput } from 'shiki/core'; + +// This file has been converted from https://github.com/NVIDIA/cuda-cpp-grammar/blob/master/syntaxes/cuda-cpp.tmLanguage.json +// If you want to provide a fix or improvement, please create a pull request against the original repository. +// Once accepted there, we are happy to receive an update request. +// version: https://github.com/NVIDIA/cuda-cpp-grammar/commit/81e88eaec5170aa8585736c63627c73e3589998c +export const cudaCpp: LanguageInput = { + name: 'CUDA C++', + scopeName: 'source.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#constructor_root', + }, + { + include: '#destructor_root', + }, + { + include: '#function_definition', + }, + { + include: '#operator_overload', + }, + { + include: '#using_namespace', + }, + { + include: '#type_alias', + }, + { + include: '#using_name', + }, + { + include: '#namespace_alias', + }, + { + include: '#namespace_block', + }, + { + include: '#extern_block', + }, + { + include: '#typedef_class', + }, + { + include: '#typedef_struct', + }, + { + include: '#typedef_union', + }, + { + include: '#misc_keywords', + }, + { + include: '#standard_declares', + }, + { + include: '#class_block', + }, + { + include: '#struct_block', + }, + { + include: '#union_block', + }, + { + include: '#enum_block', + }, + { + include: '#template_isolated_definition', + }, + { + include: '#template_definition', + }, + { + include: '#access_control_keywords', + }, + { + include: '#block', + }, + { + include: '#static_assert', + }, + { + include: '#assembly', + }, + { + include: '#function_pointer', + }, + { + include: '#evaluation_context', + }, + ], + repository: { + $self: {}, + $base: {}, + access_control_keywords: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(((?:(?:protected)|(?:private)|(?:public)))(?:(?:\\s)+)?(:))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'storage.type.modifier.access.control.$4.cuda-cpp', + }, + '4': {}, + '5': { + name: 'punctuation.separator.colon.access.control.cuda-cpp', + }, + }, + }, + alignas_attribute: { + begin: 'alignas\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.attribute.begin.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.attribute.end.cuda-cpp', + }, + }, + name: 'support.other.attribute.cuda-cpp', + patterns: [ + { + include: '#attributes_context', + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: {}, + endCaptures: {}, + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#string_context', + }, + ], + }, + { + match: '(using)(?:\\s)+((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.alignas.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.alignas.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.alignas.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.alignas', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + alignof_operator: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.alignof.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.alignof.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.alignof.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.alignof', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + assembly: { + begin: '(\\b(?:__asm__|asm)\\b)(?:(?:\\s)+)?((?:volatile)?)', + end: '(?!\\G)', + beginCaptures: { + '1': { + name: 'storage.type.asm.cuda-cpp', + }, + '2': { + name: 'storage.modifier.cuda-cpp', + }, + }, + endCaptures: {}, + name: 'meta.asm.cuda-cpp', + patterns: [ + { + match: '^((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:\\n)|$)', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + include: '#comments', + }, + { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parens.begin.bracket.round.assembly.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parens.end.bracket.round.assembly.cuda-cpp', + }, + }, + patterns: [ + { + begin: '(R?)(")', + end: '"', + beginCaptures: { + '1': { + name: 'meta.encoding.cuda-cpp', + }, + '2': { + name: 'punctuation.definition.string.begin.assembly.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.string.end.assembly.cuda-cpp', + }, + }, + name: 'string.quoted.double.cuda-cpp', + contentName: 'meta.embedded.assembly', + patterns: [ + { + include: 'source.asm', + }, + { + include: 'source.x86', + }, + { + include: 'source.x86_64', + }, + { + include: 'source.arm', + }, + { + include: '#backslash_escapes', + }, + { + include: '#string_escaped_char', + }, + ], + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parens.begin.bracket.round.assembly.inner.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parens.end.bracket.round.assembly.inner.cuda-cpp', + }, + }, + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + match: '\\[((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.other.asm.label.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: ':', + name: 'punctuation.separator.delimiter.colon.assembly.cuda-cpp', + }, + { + include: '#comments', + }, + ], + }, + ], + }, + assignment_operator: { + match: '\\=', + name: 'keyword.operator.assignment.cuda-cpp', + }, + attributes_context: { + patterns: [ + { + include: '#cpp_attributes', + }, + { + include: '#gcc_attributes', + }, + { + include: '#ms_attributes', + }, + { + include: '#alignas_attribute', + }, + ], + }, + backslash_escapes: { + match: '(?x)\\\\ (\n\\\\\t\t\t |\n[abefnprtv\'"?] |\n[0-3][0-7]{,2}\t |\n[4-7]\\d?\t\t|\nx[a-fA-F0-9]{,2} |\nu[a-fA-F0-9]{,4} |\nU[a-fA-F0-9]{,8} )', + name: 'constant.character.escape', + }, + block: { + begin: '{', + end: '}|(?=\\s*#\\s*(?:elif|else|endif)\\b)', + beginCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.cuda-cpp', + }, + }, + name: 'meta.block.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + block_comment: { + begin: '\\s*+(\\/\\*)', + end: '\\*\\/', + beginCaptures: { + '1': { + name: 'punctuation.definition.comment.begin.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.comment.end.cuda-cpp', + }, + }, + name: 'comment.block.cuda-cpp', + }, + builtin_storage_type_initilizer: { + begin: '(?:\\s)*+(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?={)|(?:((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:(?!:)))?)', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.class.cuda-cpp', + }, + '1': { + name: 'storage.type.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=:|{|$)', + captures: { + '1': { + name: 'entity.name.type.class.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: 'DLLEXPORT', + name: 'entity.name.other.preprocessor.macro.predefined.DLLEXPORT.cuda-cpp', + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$0.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '20': { + name: 'punctuation.separator.colon.inheritance.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.class.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.class.cuda-cpp', + }, + }, + name: 'meta.head.class.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#inheritance_context', + }, + { + include: '#template_call_range', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.class.cuda-cpp', + }, + }, + name: 'meta.body.class.cuda-cpp', + patterns: [ + { + include: '#function_pointer', + }, + { + include: '#static_assert', + }, + { + include: '#constructor_inline', + }, + { + include: '#destructor_inline', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.class.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + class_declare: { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.class.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.class.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + comma: { + match: ',', + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + comma_in_template_argument: { + match: ',', + name: 'punctuation.separator.delimiter.comma.template.argument.cuda-cpp', + }, + comments: { + patterns: [ + { + begin: '^(?:(?:\\s)+)?+(\\/\\/[!\\/]+)', + end: '(?<=\\n)(?|%|"|\\.|=|::|\\||\\-\\-|\\-\\-\\-)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '((?<=[\\s*!\\/])[\\\\@](?:a|em|e))(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.italic.doxygen.cuda-cpp', + }, + }, + }, + { + match: '((?<=[\\s*!\\/])[\\\\@]b)(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.bold.doxygen.cuda-cpp', + }, + }, + }, + { + match: '((?<=[\\s*!\\/])[\\\\@](?:c|p))(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.inline.raw.string.cuda-cpp', + }, + }, + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:a|anchor|b|c|cite|copybrief|copydetail|copydoc|def|dir|dontinclude|e|em|emoji|enum|example|extends|file|idlexcept|implements|include|includedoc|includelineno|latexinclude|link|memberof|namespace|p|package|ref|refitem|related|relates|relatedalso|relatesalso|verbinclude)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:addindex|addtogroup|category|class|defgroup|diafile|dotfile|elseif|fn|headerfile|if|ifnot|image|ingroup|interface|line|mainpage|mscfile|name|overload|page|property|protocol|section|skip|skipline|snippet|snippetdoc|snippetlineno|struct|subpage|subsection|subsubsection|typedef|union|until|vhdlflow|weakgroup)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '((?<=[\\s*!\\/])[\\\\@]param)(?:\\s*\\[((?:,?(?:(?:\\s)+)?(?:in|out)(?:(?:\\s)+)?)+)\\])?(?:\\s)+(\\b\\w+\\b)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + patterns: [ + { + match: 'in|out', + name: 'keyword.other.parameter.direction.$0.cuda-cpp', + }, + ], + }, + '3': { + name: 'variable.parameter.cuda-cpp', + }, + }, + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:arg|attention|author|authors|brief|bug|copyright|date|deprecated|details|exception|invariant|li|note|par|paragraph|param|post|pre|remark|remarks|result|return|returns|retval|sa|see|short|since|test|throw|throws|todo|tparam|version|warning|xrefitem)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:code|cond|docbookonly|dot|htmlonly|internal|latexonly|link|manonly|msc|parblock|rtfonly|secreflist|startuml|verbatim|xmlonly|endcode|endcond|enddocbookonly|enddot|endhtmlonly|endinternal|endlatexonly|endlink|endmanonly|endmsc|endparblock|endrtfonly|endsecreflist|enduml|endverbatim|endxmlonly)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?:\\b[A-Z]+:|@[a-z_]+:)', + name: 'storage.type.class.gtkdoc.cuda-cpp', + }, + ], + }, + { + match: '(\\/\\*[!*]+(?=\\s))(.+)([!*]*\\*\\/)', + captures: { + '1': { + name: 'punctuation.definition.comment.begin.documentation.cuda-cpp', + }, + '2': { + patterns: [ + { + match: '(?<=[\\s*!\\/])[\\\\@](?:callergraph|callgraph|else|endif|f\\$|f\\[|f\\]|hidecallergraph|hidecallgraph|hiderefby|hiderefs|hideinitializer|htmlinclude|n|nosubgrouping|private|privatesection|protected|protectedsection|public|publicsection|pure|showinitializer|showrefby|showrefs|tableofcontents|\\$|\\#|<|>|%|"|\\.|=|::|\\||\\-\\-|\\-\\-\\-)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '((?<=[\\s*!\\/])[\\\\@](?:a|em|e))(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.italic.doxygen.cuda-cpp', + }, + }, + }, + { + match: '((?<=[\\s*!\\/])[\\\\@]b)(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.bold.doxygen.cuda-cpp', + }, + }, + }, + { + match: '((?<=[\\s*!\\/])[\\\\@](?:c|p))(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.inline.raw.string.cuda-cpp', + }, + }, + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:a|anchor|b|c|cite|copybrief|copydetail|copydoc|def|dir|dontinclude|e|em|emoji|enum|example|extends|file|idlexcept|implements|include|includedoc|includelineno|latexinclude|link|memberof|namespace|p|package|ref|refitem|related|relates|relatedalso|relatesalso|verbinclude)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:addindex|addtogroup|category|class|defgroup|diafile|dotfile|elseif|fn|headerfile|if|ifnot|image|ingroup|interface|line|mainpage|mscfile|name|overload|page|property|protocol|section|skip|skipline|snippet|snippetdoc|snippetlineno|struct|subpage|subsection|subsubsection|typedef|union|until|vhdlflow|weakgroup)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '((?<=[\\s*!\\/])[\\\\@]param)(?:\\s*\\[((?:,?(?:(?:\\s)+)?(?:in|out)(?:(?:\\s)+)?)+)\\])?(?:\\s)+(\\b\\w+\\b)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + patterns: [ + { + match: 'in|out', + name: 'keyword.other.parameter.direction.$0.cuda-cpp', + }, + ], + }, + '3': { + name: 'variable.parameter.cuda-cpp', + }, + }, + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:arg|attention|author|authors|brief|bug|copyright|date|deprecated|details|exception|invariant|li|note|par|paragraph|param|post|pre|remark|remarks|result|return|returns|retval|sa|see|short|since|test|throw|throws|todo|tparam|version|warning|xrefitem)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:code|cond|docbookonly|dot|htmlonly|internal|latexonly|link|manonly|msc|parblock|rtfonly|secreflist|startuml|verbatim|xmlonly|endcode|endcond|enddocbookonly|enddot|endhtmlonly|endinternal|endlatexonly|endlink|endmanonly|endmsc|endparblock|endrtfonly|endsecreflist|enduml|endverbatim|endxmlonly)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?:\\b[A-Z]+:|@[a-z_]+:)', + name: 'storage.type.class.gtkdoc.cuda-cpp', + }, + ], + }, + '3': { + name: 'punctuation.definition.comment.end.documentation.cuda-cpp', + }, + }, + name: 'comment.block.documentation.cuda-cpp', + }, + { + begin: '(?:(?:\\s)+)?+\\/\\*[!*]+(?:(?:(?:\\n)|$)|(?=\\s))', + end: '[!*]*\\*\\/', + beginCaptures: { + '0': { + name: 'punctuation.definition.comment.begin.documentation.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.comment.end.documentation.cuda-cpp', + }, + }, + name: 'comment.block.documentation.cuda-cpp', + patterns: [ + { + match: '(?<=[\\s*!\\/])[\\\\@](?:callergraph|callgraph|else|endif|f\\$|f\\[|f\\]|hidecallergraph|hidecallgraph|hiderefby|hiderefs|hideinitializer|htmlinclude|n|nosubgrouping|private|privatesection|protected|protectedsection|public|publicsection|pure|showinitializer|showrefby|showrefs|tableofcontents|\\$|\\#|<|>|%|"|\\.|=|::|\\||\\-\\-|\\-\\-\\-)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '((?<=[\\s*!\\/])[\\\\@](?:a|em|e))(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.italic.doxygen.cuda-cpp', + }, + }, + }, + { + match: '((?<=[\\s*!\\/])[\\\\@]b)(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.bold.doxygen.cuda-cpp', + }, + }, + }, + { + match: '((?<=[\\s*!\\/])[\\\\@](?:c|p))(?:\\s)+(\\S+)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + name: 'markup.inline.raw.string.cuda-cpp', + }, + }, + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:a|anchor|b|c|cite|copybrief|copydetail|copydoc|def|dir|dontinclude|e|em|emoji|enum|example|extends|file|idlexcept|implements|include|includedoc|includelineno|latexinclude|link|memberof|namespace|p|package|ref|refitem|related|relates|relatedalso|relatesalso|verbinclude)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:addindex|addtogroup|category|class|defgroup|diafile|dotfile|elseif|fn|headerfile|if|ifnot|image|ingroup|interface|line|mainpage|mscfile|name|overload|page|property|protocol|section|skip|skipline|snippet|snippetdoc|snippetlineno|struct|subpage|subsection|subsubsection|typedef|union|until|vhdlflow|weakgroup)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '((?<=[\\s*!\\/])[\\\\@]param)(?:\\s*\\[((?:,?(?:(?:\\s)+)?(?:in|out)(?:(?:\\s)+)?)+)\\])?(?:\\s)+(\\b\\w+\\b)', + captures: { + '1': { + name: 'storage.type.class.doxygen.cuda-cpp', + }, + '2': { + patterns: [ + { + match: 'in|out', + name: 'keyword.other.parameter.direction.$0.cuda-cpp', + }, + ], + }, + '3': { + name: 'variable.parameter.cuda-cpp', + }, + }, + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:arg|attention|author|authors|brief|bug|copyright|date|deprecated|details|exception|invariant|li|note|par|paragraph|param|post|pre|remark|remarks|result|return|returns|retval|sa|see|short|since|test|throw|throws|todo|tparam|version|warning|xrefitem)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?<=[\\s*!\\/])[\\\\@](?:code|cond|docbookonly|dot|htmlonly|internal|latexonly|link|manonly|msc|parblock|rtfonly|secreflist|startuml|verbatim|xmlonly|endcode|endcond|enddocbookonly|enddot|endhtmlonly|endinternal|endlatexonly|endlink|endmanonly|endmsc|endparblock|endrtfonly|endsecreflist|enduml|endverbatim|endxmlonly)\\b(?:\\{[^}]*\\})?', + name: 'storage.type.class.doxygen.cuda-cpp', + }, + { + match: '(?:\\b[A-Z]+:|@[a-z_]+:)', + name: 'storage.type.class.gtkdoc.cuda-cpp', + }, + ], + }, + { + include: '#emacs_file_banner', + }, + { + include: '#block_comment', + }, + { + include: '#line_comment', + }, + { + include: '#invalid_comment_end', + }, + ], + }, + constructor_inline: { + begin: '^((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:(?:(?:__forceinline__)|(?:__noinline__)|(?:__global__)|(?:__device__)|(?:constexpr)|(?:explicit)|(?:__host__)|(?:mutable)|(?:virtual)|(?:inline)|(?:friend))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:__cdecl|__clrcall|__stdcall|__fastcall|__thiscall|__vectorcall)?)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.function.definition.special.constructor.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + patterns: [ + { + include: '#functional_specifiers_pre_parameters', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '12': { + name: 'comment.block.cuda-cpp', + }, + '13': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '14': { + name: 'storage.type.modifier.calling-convention.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '17': { + name: 'comment.block.cuda-cpp', + }, + '18': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '19': { + name: 'entity.name.function.constructor.cuda-cpp entity.name.function.definition.special.constructor.cuda-cpp', + }, + }, + endCaptures: {}, + name: 'meta.function.definition.special.constructor.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.function.definition.special.constructor.cuda-cpp', + }, + }, + name: 'meta.head.function.definition.special.constructor.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + match: '(\\=)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(default)|(delete))', + captures: { + '1': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'keyword.other.default.constructor.cuda-cpp', + }, + '7': { + name: 'keyword.other.delete.constructor.cuda-cpp', + }, + }, + }, + { + include: '#functional_specifiers_pre_parameters', + }, + { + begin: ':', + end: '(?=\\{)', + beginCaptures: { + '0': { + name: 'punctuation.separator.initializers.cuda-cpp', + }, + }, + endCaptures: {}, + patterns: [ + { + begin: '((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'entity.name.function.call.initializer.cuda-cpp', + }, + '2': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '3': {}, + '4': { + name: 'punctuation.section.arguments.begin.bracket.round.function.call.initializer.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.function.call.initializer.cuda-cpp', + }, + }, + contentName: 'meta.parameter.initialization', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '((?|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.function.definition.special.constructor.cuda-cpp', + }, + }, + name: 'meta.body.function.definition.special.constructor.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.function.definition.special.constructor.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + constructor_root: { + begin: '\\s*+((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:__cdecl|__clrcall|__stdcall|__fastcall|__thiscall|__vectorcall)?)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)(?:\\s)*+)?::)*+)(((?>(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))::((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\14((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\())', + end: '(?:(?<=\\}|%>|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.function.definition.special.constructor.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'storage.type.modifier.calling-convention.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.constructor.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(default)|(delete))', + captures: { + '1': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'keyword.other.default.constructor.cuda-cpp', + }, + '7': { + name: 'keyword.other.delete.constructor.cuda-cpp', + }, + }, + }, + { + include: '#functional_specifiers_pre_parameters', + }, + { + begin: ':', + end: '(?=\\{)', + beginCaptures: { + '0': { + name: 'punctuation.separator.initializers.cuda-cpp', + }, + }, + endCaptures: {}, + patterns: [ + { + begin: '((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'entity.name.function.call.initializer.cuda-cpp', + }, + '2': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '3': {}, + '4': { + name: 'punctuation.section.arguments.begin.bracket.round.function.call.initializer.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.function.call.initializer.cuda-cpp', + }, + }, + contentName: 'meta.parameter.initialization', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '((?|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.function.definition.special.constructor.cuda-cpp', + }, + }, + name: 'meta.body.function.definition.special.constructor.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.function.definition.special.constructor.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + control_flow_keywords: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.control.$3.cuda-cpp', + }, + }, + }, + cpp_attributes: { + begin: '\\[\\[', + end: '\\]\\]', + beginCaptures: { + '0': { + name: 'punctuation.section.attribute.begin.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.attribute.end.cuda-cpp', + }, + }, + name: 'support.other.attribute.cuda-cpp', + patterns: [ + { + include: '#attributes_context', + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: {}, + endCaptures: {}, + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#string_context', + }, + ], + }, + { + match: '(using)(?:\\s)+((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)?(?![\\w<:.]))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\{)', + end: '\\}', + beginCaptures: { + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((import))(?:(?:\\s)+)?(?:(?:(?:((<)[^>]*(>?)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=\\/\\/)))|((\\")[^\\"]*((?:\\")?)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=\\/\\/))))|(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*(?:\\.(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)*((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=(?:\\/\\/|;)))))|((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=(?:\\/\\/|;))))(?:(?:\\s)+)?(;?)', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.control.directive.import.cuda-cpp', + }, + '5': { + name: 'string.quoted.other.lt-gt.include.cuda-cpp', + }, + '6': { + name: 'punctuation.definition.string.begin.cuda-cpp', + }, + '7': { + name: 'punctuation.definition.string.end.cuda-cpp', + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + name: 'string.quoted.double.include.cuda-cpp', + }, + '11': { + name: 'punctuation.definition.string.begin.cuda-cpp', + }, + '12': { + name: 'punctuation.definition.string.end.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '15': { + name: 'entity.name.other.preprocessor.macro.include.cuda-cpp', + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '18': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '19': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '20': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '21': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '22': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.preprocessor.import.cuda-cpp', + }, + d9bc4796b0b_preprocessor_number_literal: { + match: "(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.other.decltype.cuda-cpp storage.type.decltype.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.decltype.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.decltype.cuda-cpp', + }, + }, + contentName: 'meta.arguments.decltype', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + decltype_specifier: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.other.decltype.cuda-cpp storage.type.decltype.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.decltype.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.decltype.cuda-cpp', + }, + }, + contentName: 'meta.arguments.decltype', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + default_statement: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:__cdecl|__clrcall|__stdcall|__fastcall|__thiscall|__vectorcall)?)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:(?:(?:__forceinline__)|(?:__noinline__)|(?:__global__)|(?:__device__)|(?:constexpr)|(?:explicit)|(?:__host__)|(?:mutable)|(?:virtual)|(?:inline)|(?:friend))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*)(~(?|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.function.definition.special.member.destructor.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '7': { + name: 'comment.block.cuda-cpp', + }, + '8': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '9': { + name: 'storage.type.modifier.calling-convention.cuda-cpp', + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '12': { + name: 'comment.block.cuda-cpp', + }, + '13': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '14': { + patterns: [ + { + include: '#functional_specifiers_pre_parameters', + }, + ], + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '17': { + name: 'comment.block.cuda-cpp', + }, + '18': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '19': { + name: 'entity.name.function.destructor.cuda-cpp entity.name.function.definition.special.member.destructor.cuda-cpp', + }, + }, + endCaptures: {}, + name: 'meta.function.definition.special.member.destructor.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.function.definition.special.member.destructor.cuda-cpp', + }, + }, + name: 'meta.head.function.definition.special.member.destructor.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + match: '(\\=)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(default)|(delete))', + captures: { + '1': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'keyword.other.default.constructor.cuda-cpp', + }, + '7': { + name: 'keyword.other.delete.constructor.cuda-cpp', + }, + }, + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parameters.begin.bracket.round.special.member.destructor.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parameters.end.bracket.round.special.member.destructor.cuda-cpp', + }, + }, + contentName: 'meta.function.definition.parameters.special.member.destructor', + patterns: [], + }, + { + match: '((?:(?:final)|(?:override)))+', + captures: { + '1': { + name: 'keyword.operator.wordlike.cuda-cpp keyword.operator.$1.cuda-cpp', + }, + }, + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.function.definition.special.member.destructor.cuda-cpp', + }, + }, + name: 'meta.body.function.definition.special.member.destructor.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.function.definition.special.member.destructor.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + destructor_root: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:__cdecl|__clrcall|__stdcall|__fastcall|__thiscall|__vectorcall)?)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)(?:\\s)*+)?::)*+)(((?>(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))::((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))~\\14((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\())', + end: '(?:(?<=\\}|%>|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.function.definition.special.member.destructor.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'storage.type.modifier.calling-convention.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.destructor.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(default)|(delete))', + captures: { + '1': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'keyword.other.default.constructor.cuda-cpp', + }, + '7': { + name: 'keyword.other.delete.constructor.cuda-cpp', + }, + }, + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parameters.begin.bracket.round.special.member.destructor.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parameters.end.bracket.round.special.member.destructor.cuda-cpp', + }, + }, + contentName: 'meta.function.definition.parameters.special.member.destructor', + patterns: [], + }, + { + match: '((?:(?:final)|(?:override)))+', + captures: { + '1': { + name: 'keyword.operator.wordlike.cuda-cpp keyword.operator.$1.cuda-cpp', + }, + }, + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.function.definition.special.member.destructor.cuda-cpp', + }, + }, + name: 'meta.body.function.definition.special.member.destructor.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.function.definition.special.member.destructor.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + diagnostic: { + begin: '(^((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?((?:error|warning)))\\b(?:(?:\\s)+)?', + end: '(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)(?:\\s)*+)?(::))?(?:(?:\\s)+)?((?|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.enum.cuda-cpp', + }, + '1': { + name: 'storage.type.enum.cuda-cpp', + }, + '2': { + name: 'storage.type.enum.enum-key.$2.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '4': { + name: 'entity.name.type.enum.cuda-cpp', + }, + '5': { + name: 'punctuation.separator.colon.type-specifier.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#scope_resolution_inner_generated', + }, + ], + }, + '7': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + '8': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '9': {}, + '10': { + name: 'entity.name.scope-resolution.cuda-cpp', + }, + '11': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '12': {}, + '13': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + '14': { + name: 'storage.type.integral.$14.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.enum.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.enum.cuda-cpp', + }, + }, + name: 'meta.head.enum.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.enum.cuda-cpp', + }, + }, + name: 'meta.body.enum.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#enumerator_list', + }, + { + include: '#comments', + }, + { + include: '#comma', + }, + { + include: '#semicolon', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.enum.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + enum_declare: { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.enum.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.enum.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + enumerator_list: { + match: "((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.control.exception.$3.cuda-cpp', + }, + }, + }, + extern_block: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(extern)(?=\\s*\\")', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.extern.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'storage.type.extern.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.extern.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.extern.cuda-cpp', + }, + }, + name: 'meta.head.extern.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.extern.cuda-cpp', + }, + }, + name: 'meta.body.extern.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.extern.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + { + include: '$self', + }, + ], + }, + function_body_context: { + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#using_namespace', + }, + { + include: '#type_alias', + }, + { + include: '#using_name', + }, + { + include: '#namespace_alias', + }, + { + include: '#typedef_class', + }, + { + include: '#typedef_struct', + }, + { + include: '#typedef_union', + }, + { + include: '#misc_keywords', + }, + { + include: '#standard_declares', + }, + { + include: '#class_block', + }, + { + include: '#struct_block', + }, + { + include: '#union_block', + }, + { + include: '#enum_block', + }, + { + include: '#access_control_keywords', + }, + { + include: '#block', + }, + { + include: '#static_assert', + }, + { + include: '#assembly', + }, + { + include: '#function_pointer', + }, + { + include: '#switch_statement', + }, + { + include: '#goto_statement', + }, + { + include: '#evaluation_context', + }, + { + include: '#label', + }, + ], + }, + function_call: { + begin: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<11>?)+>)(?:\\s)*+)?::)*\\s*+)((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)\\b(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<11>?)+>)(?:\\s)*+)?(\\()', + end: '\\)', + beginCaptures: { + '1': { + patterns: [ + { + include: '#scope_resolution_function_call_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.call.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.function.call.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '11': {}, + '12': { + name: 'punctuation.section.arguments.begin.bracket.round.function.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.function.call.cuda-cpp', + }, + }, + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + function_definition: { + begin: '(?:(?:^|\\G|(?<=;|\\}))|(?<=>))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*)(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<60>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<60>?)+>)?(?![\\w<:.]))(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:__cdecl|__clrcall|__stdcall|__fastcall|__thiscall|__vectorcall)?)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<60>?)+>)(?:\\s)*+)?::)*\\s*+)((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)\\b(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\()', + end: '(?:(?<=\\}|%>|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.function.definition.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'storage.type.template.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.modifier.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'storage.modifier.$12.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '15': { + name: 'comment.block.cuda-cpp', + }, + '16': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '17': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '18': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '21': { + name: 'comment.block.cuda-cpp', + }, + '22': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '23': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '24': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '25': { + name: 'comment.block.cuda-cpp', + }, + '26': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '27': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '36': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '37': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '38': { + name: 'comment.block.cuda-cpp', + }, + '39': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '40': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '41': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '42': { + name: 'comment.block.cuda-cpp', + }, + '43': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '44': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '45': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '46': { + name: 'comment.block.cuda-cpp', + }, + '47': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '48': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '49': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '50': { + name: 'comment.block.cuda-cpp', + }, + '51': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '52': { + name: 'storage.type.modifier.calling-convention.cuda-cpp', + }, + '53': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '54': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '55': { + name: 'comment.block.cuda-cpp', + }, + '56': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '57': { + patterns: [ + { + include: '#scope_resolution_function_definition_inner_generated', + }, + ], + }, + '58': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.cuda-cpp', + }, + '59': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '60': {}, + '61': { + name: 'entity.name.function.definition.cuda-cpp', + }, + '62': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '63': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '64': { + name: 'comment.block.cuda-cpp', + }, + '65': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + endCaptures: {}, + name: 'meta.function.definition.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.function.definition.cuda-cpp', + }, + }, + name: 'meta.head.function.definition.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parameters.begin.bracket.round.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parameters.end.bracket.round.cuda-cpp', + }, + }, + contentName: 'meta.function.definition.parameters', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#parameter_or_maybe_value', + }, + { + include: '#comma', + }, + { + include: '#evaluation_context', + }, + ], + }, + { + match: '(?<=^|\\))(?:(?:\\s)+)?(->)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<23>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<23>?)+>)?(?![\\w<:.]))', + captures: { + '1': { + name: 'punctuation.definition.function.return-type.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '10': { + name: 'comment.block.cuda-cpp', + }, + '11': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.function.definition.cuda-cpp', + }, + }, + name: 'meta.body.function.definition.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.function.definition.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + function_parameter_context: { + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#parameter', + }, + { + include: '#comma', + }, + ], + }, + function_pointer: { + begin: '(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)?(?![\\w<:.]))(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()(\\*)(?:(?:\\s)+)?((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)?)(?:(?:\\s)+)?(?:(\\[)(\\w*)(\\])(?:(?:\\s)+)?)*(\\))(?:(?:\\s)+)?(\\()', + end: '(\\))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=[{=,);>]|\\n)(?!\\()', + beginCaptures: { + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '20': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '21': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '22': { + name: 'comment.block.cuda-cpp', + }, + '23': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '24': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '25': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '26': { + name: 'comment.block.cuda-cpp', + }, + '27': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '28': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '29': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '30': { + name: 'comment.block.cuda-cpp', + }, + '31': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '32': { + name: 'punctuation.section.parens.begin.bracket.round.function.pointer.cuda-cpp', + }, + '33': { + name: 'punctuation.definition.function.pointer.dereference.cuda-cpp', + }, + '34': { + name: 'variable.other.definition.pointer.function.cuda-cpp', + }, + '35': { + name: 'punctuation.definition.begin.bracket.square.cuda-cpp', + }, + '36': { + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + '37': { + name: 'punctuation.definition.end.bracket.square.cuda-cpp', + }, + '38': { + name: 'punctuation.section.parens.end.bracket.round.function.pointer.cuda-cpp', + }, + '39': { + name: 'punctuation.section.parameters.begin.bracket.round.function.pointer.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.section.parameters.end.bracket.round.function.pointer.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + patterns: [ + { + include: '#function_parameter_context', + }, + ], + }, + function_pointer_parameter: { + begin: '(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)?(?![\\w<:.]))(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()(\\*)(?:(?:\\s)+)?((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)?)(?:(?:\\s)+)?(?:(\\[)(\\w*)(\\])(?:(?:\\s)+)?)*(\\))(?:(?:\\s)+)?(\\()', + end: '(\\))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=[{=,);>]|\\n)(?!\\()', + beginCaptures: { + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '20': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '21': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '22': { + name: 'comment.block.cuda-cpp', + }, + '23': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '24': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '25': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '26': { + name: 'comment.block.cuda-cpp', + }, + '27': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '28': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '29': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '30': { + name: 'comment.block.cuda-cpp', + }, + '31': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '32': { + name: 'punctuation.section.parens.begin.bracket.round.function.pointer.cuda-cpp', + }, + '33': { + name: 'punctuation.definition.function.pointer.dereference.cuda-cpp', + }, + '34': { + name: 'variable.parameter.pointer.function.cuda-cpp', + }, + '35': { + name: 'punctuation.definition.begin.bracket.square.cuda-cpp', + }, + '36': { + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + '37': { + name: 'punctuation.definition.end.bracket.square.cuda-cpp', + }, + '38': { + name: 'punctuation.section.parens.end.bracket.round.function.pointer.cuda-cpp', + }, + '39': { + name: 'punctuation.section.parameters.begin.bracket.round.function.pointer.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.section.parameters.end.bracket.round.function.pointer.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + patterns: [ + { + include: '#function_parameter_context', + }, + ], + }, + functional_specifiers_pre_parameters: { + match: '(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)', + captures: { + '1': { + name: 'keyword.control.goto.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.label.call.cuda-cpp', + }, + }, + }, + identifier: { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + }, + include: { + match: '^((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((#)(?:(?:\\s)+)?((?:include|include_next))\\b)(?:(?:\\s)+)?(?:(?:(?:((<)[^>]*(>?)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=\\/\\/)))|((\\")[^\\"]*((?:\\")?)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=\\/\\/))))|(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*(?:\\.(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)*((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=(?:\\/\\/|;)))))|((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:\\n)|$)|(?=(?:\\/\\/|;))))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.control.directive.$5.cuda-cpp', + }, + '4': { + name: 'punctuation.definition.directive.cuda-cpp', + }, + '6': { + name: 'string.quoted.other.lt-gt.include.cuda-cpp', + }, + '7': { + name: 'punctuation.definition.string.begin.cuda-cpp', + }, + '8': { + name: 'punctuation.definition.string.end.cuda-cpp', + }, + '9': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '10': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '11': { + name: 'string.quoted.double.include.cuda-cpp', + }, + '12': { + name: 'punctuation.definition.string.begin.cuda-cpp', + }, + '13': { + name: 'punctuation.definition.string.end.cuda-cpp', + }, + '14': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '15': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '16': { + name: 'entity.name.other.preprocessor.macro.include.cuda-cpp', + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '21': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '22': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + name: 'meta.preprocessor.include.cuda-cpp', + }, + inheritance_context: { + patterns: [ + { + include: '#ever_present_context', + }, + { + match: ',', + name: 'punctuation.separator.delimiter.comma.inheritance.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)(?:\\s)*+)?::)*+)?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)?(?![\\w<:.]))', + captures: { + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': {}, + }, + }, + ], + }, + inline_builtin_storage_type: { + match: '(?:\\s)*+(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:)', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'entity.name.label.cuda-cpp', + }, + '4': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '5': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '6': { + name: 'punctuation.separator.label.cuda-cpp', + }, + }, + }, + lambdas: { + begin: '(?:(?<=[^\\s]|^)(?])|(?<=\\Wreturn|^return))(?:(?:\\s)+)?(\\[(?!\\[| *+"| *+\\d))((?:[^\\[\\]]|((??)++\\]))*+)(\\](?!((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))[\\[\\];]))', + end: '(?<=[;}])', + beginCaptures: { + '1': { + name: 'punctuation.definition.capture.begin.lambda.cuda-cpp', + }, + '2': { + name: 'meta.lambda.capture.cuda-cpp', + patterns: [ + { + include: '#the_this_keyword', + }, + { + match: '((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?=\\]|\\z|$)|(,))|(\\=))', + captures: { + '1': { + name: 'variable.parameter.capture.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + '7': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + }, + }, + { + include: '#evaluation_context', + }, + ], + }, + '3': {}, + '4': { + name: 'punctuation.definition.capture.end.lambda.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '7': { + name: 'comment.block.cuda-cpp', + }, + '8': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + endCaptures: {}, + patterns: [ + { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.definition.parameters.begin.lambda.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.parameters.end.lambda.cuda-cpp', + }, + }, + name: 'meta.function.definition.parameters.lambda.cuda-cpp', + patterns: [ + { + include: '#function_parameter_context', + }, + ], + }, + { + match: '(?)((?:.+?(?=\\{|$))?)', + captures: { + '1': { + name: 'punctuation.definition.lambda.return-type.cuda-cpp', + }, + '2': { + name: 'storage.type.return-type.lambda.cuda-cpp', + }, + }, + }, + { + begin: '\\{', + end: '\\}', + beginCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.lambda.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.lambda.cuda-cpp', + }, + }, + name: 'meta.function.definition.body.lambda.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + language_constants: { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?line\\b', + end: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?define\\b)(?:(?:\\s)+)?((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?\\*|->)))((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*(?:(?:\\s)+)?(?:(?:\\.\\*|\\.)|(?:->\\*|->))(?:(?:\\s)+)?)*)(?:(?:\\s)+)?(\\b(?!uint_least16_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_least32_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_least64_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_least16_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_least32_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_least64_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_least8_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_fast16_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_fast32_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_fast64_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_least8_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_fast16_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_fast32_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_fast64_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint_fast8_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|suseconds_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int_fast8_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|useconds_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulonglong1[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulonglong2[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulonglong3[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulonglong4[^Pattern.new(\n match: \\/\\w\\/,\n)]|blksize_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|in_addr_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|in_port_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uintptr_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uintmax_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uintmax_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uintmax_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|longlong1[^Pattern.new(\n match: \\/\\w\\/,\n)]|longlong2[^Pattern.new(\n match: \\/\\w\\/,\n)]|longlong3[^Pattern.new(\n match: \\/\\w\\/,\n)]|longlong4[^Pattern.new(\n match: \\/\\w\\/,\n)]|unsigned[^Pattern.new(\n match: \\/\\w\\/,\n)]|u_quad_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|blkcnt_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint16_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint32_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint64_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|intptr_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|intmax_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|intmax_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|wchar_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|u_short[^Pattern.new(\n match: \\/\\w\\/,\n)]|qaddr_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|caddr_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|daddr_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|fixpt_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|nlink_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|segsz_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|swblk_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|clock_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|ssize_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int16_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int32_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int64_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint8_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|ushort1[^Pattern.new(\n match: \\/\\w\\/,\n)]|ushort2[^Pattern.new(\n match: \\/\\w\\/,\n)]|ushort3[^Pattern.new(\n match: \\/\\w\\/,\n)]|ushort4[^Pattern.new(\n match: \\/\\w\\/,\n)]|double1[^Pattern.new(\n match: \\/\\w\\/,\n)]|double2[^Pattern.new(\n match: \\/\\w\\/,\n)]|double3[^Pattern.new(\n match: \\/\\w\\/,\n)]|double4[^Pattern.new(\n match: \\/\\w\\/,\n)]|signed[^Pattern.new(\n match: \\/\\w\\/,\n)]|double[^Pattern.new(\n match: \\/\\w\\/,\n)]|u_char[^Pattern.new(\n match: \\/\\w\\/,\n)]|u_long[^Pattern.new(\n match: \\/\\w\\/,\n)]|ushort[^Pattern.new(\n match: \\/\\w\\/,\n)]|quad_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|mode_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|size_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|time_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int8_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uchar1[^Pattern.new(\n match: \\/\\w\\/,\n)]|uchar2[^Pattern.new(\n match: \\/\\w\\/,\n)]|uchar3[^Pattern.new(\n match: \\/\\w\\/,\n)]|uchar4[^Pattern.new(\n match: \\/\\w\\/,\n)]|short1[^Pattern.new(\n match: \\/\\w\\/,\n)]|short2[^Pattern.new(\n match: \\/\\w\\/,\n)]|short3[^Pattern.new(\n match: \\/\\w\\/,\n)]|short4[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulong4[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulong1[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulong2[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulong3[^Pattern.new(\n match: \\/\\w\\/,\n)]|ulong4[^Pattern.new(\n match: \\/\\w\\/,\n)]|float1[^Pattern.new(\n match: \\/\\w\\/,\n)]|float2[^Pattern.new(\n match: \\/\\w\\/,\n)]|float3[^Pattern.new(\n match: \\/\\w\\/,\n)]|float4[^Pattern.new(\n match: \\/\\w\\/,\n)]|short[^Pattern.new(\n match: \\/\\w\\/,\n)]|float[^Pattern.new(\n match: \\/\\w\\/,\n)]|u_int[^Pattern.new(\n match: \\/\\w\\/,\n)]|div_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|dev_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|gid_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|ino_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|key_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|pid_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|off_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|uid_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|char1[^Pattern.new(\n match: \\/\\w\\/,\n)]|char2[^Pattern.new(\n match: \\/\\w\\/,\n)]|char3[^Pattern.new(\n match: \\/\\w\\/,\n)]|char4[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint1[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint2[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint3[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint4[^Pattern.new(\n match: \\/\\w\\/,\n)]|long1[^Pattern.new(\n match: \\/\\w\\/,\n)]|long2[^Pattern.new(\n match: \\/\\w\\/,\n)]|long3[^Pattern.new(\n match: \\/\\w\\/,\n)]|auto[^Pattern.new(\n match: \\/\\w\\/,\n)]|void[^Pattern.new(\n match: \\/\\w\\/,\n)]|char[^Pattern.new(\n match: \\/\\w\\/,\n)]|long[^Pattern.new(\n match: \\/\\w\\/,\n)]|bool[^Pattern.new(\n match: \\/\\w\\/,\n)]|uint[^Pattern.new(\n match: \\/\\w\\/,\n)]|id_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|id_t[^Pattern.new(\n match: \\/\\w\\/,\n)]|int1[^Pattern.new(\n match: \\/\\w\\/,\n)]|int2[^Pattern.new(\n match: \\/\\w\\/,\n)]|int3[^Pattern.new(\n match: \\/\\w\\/,\n)]|int4[^Pattern.new(\n match: \\/\\w\\/,\n)]|dim3[^Pattern.new(\n match: \\/\\w\\/,\n)]|int[^Pattern.new(\n match: \\/\\w\\/,\n)])(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b(?!\\())', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'variable.language.this.cuda-cpp', + }, + '4': { + name: 'variable.other.object.access.cuda-cpp', + }, + '5': { + name: 'punctuation.separator.dot-access.cuda-cpp', + }, + '6': { + name: 'punctuation.separator.pointer-access.cuda-cpp', + }, + '7': { + patterns: [ + { + match: '(?<=(?:\\.\\*|\\.|->|->\\*))(?:(?:\\s)+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?\\*|->)))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.language.this.cuda-cpp', + }, + '6': { + name: 'variable.other.object.property.cuda-cpp', + }, + '7': { + name: 'punctuation.separator.dot-access.cuda-cpp', + }, + '8': { + name: 'punctuation.separator.pointer-access.cuda-cpp', + }, + }, + }, + { + match: '(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?\\*|->)))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.language.this.cuda-cpp', + }, + '6': { + name: 'variable.other.object.access.cuda-cpp', + }, + '7': { + name: 'punctuation.separator.dot-access.cuda-cpp', + }, + '8': { + name: 'punctuation.separator.pointer-access.cuda-cpp', + }, + }, + }, + { + include: '#member_access', + }, + { + include: '#method_access', + }, + ], + }, + '8': { + name: 'variable.other.property.cuda-cpp', + }, + }, + }, + memory_operators: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:(?:(delete)(?:(?:\\s)+)?(\\[\\])|(delete))|(new))(?!\\w))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.operator.wordlike.cuda-cpp', + }, + '4': { + name: 'keyword.operator.delete.array.cuda-cpp', + }, + '5': { + name: 'keyword.operator.delete.array.bracket.cuda-cpp', + }, + '6': { + name: 'keyword.operator.delete.cuda-cpp', + }, + '7': { + name: 'keyword.operator.new.cuda-cpp', + }, + }, + }, + method_access: { + begin: '(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?\\*|->)))((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*(?:(?:\\s)+)?(?:(?:\\.\\*|\\.)|(?:->\\*|->))(?:(?:\\s)+)?)*)(?:(?:\\s)+)?(~?(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)(?:(?:\\s)+)?(\\()', + end: '\\)', + beginCaptures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.language.this.cuda-cpp', + }, + '6': { + name: 'variable.other.object.access.cuda-cpp', + }, + '7': { + name: 'punctuation.separator.dot-access.cuda-cpp', + }, + '8': { + name: 'punctuation.separator.pointer-access.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '(?<=(?:\\.\\*|\\.|->|->\\*))(?:(?:\\s)+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?\\*|->)))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.language.this.cuda-cpp', + }, + '6': { + name: 'variable.other.object.property.cuda-cpp', + }, + '7': { + name: 'punctuation.separator.dot-access.cuda-cpp', + }, + '8': { + name: 'punctuation.separator.pointer-access.cuda-cpp', + }, + }, + }, + { + match: '(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?\\*|->)))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.language.this.cuda-cpp', + }, + '6': { + name: 'variable.other.object.access.cuda-cpp', + }, + '7': { + name: 'punctuation.separator.dot-access.cuda-cpp', + }, + '8': { + name: 'punctuation.separator.pointer-access.cuda-cpp', + }, + }, + }, + { + include: '#member_access', + }, + { + include: '#method_access', + }, + ], + }, + '10': { + name: 'entity.name.function.member.cuda-cpp', + }, + '11': { + name: 'punctuation.section.arguments.begin.bracket.round.function.member.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.function.member.cuda-cpp', + }, + }, + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + misc_keywords: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.other.$3.cuda-cpp', + }, + }, + }, + ms_attributes: { + begin: '__declspec\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.attribute.begin.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.attribute.end.cuda-cpp', + }, + }, + name: 'support.other.attribute.cuda-cpp', + patterns: [ + { + include: '#attributes_context', + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: {}, + endCaptures: {}, + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#string_context', + }, + ], + }, + { + match: '(using)(?:\\s)+((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<8>?)+>)(?:\\s)*+)?::)*\\s*+)(?:(?:\\s)+)?((?|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.namespace.cuda-cpp', + }, + '1': { + name: 'keyword.other.namespace.definition.cuda-cpp storage.type.namespace.definition.cuda-cpp', + }, + }, + endCaptures: {}, + name: 'meta.block.namespace.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.namespace.cuda-cpp', + }, + }, + name: 'meta.head.namespace.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#attributes_context', + }, + { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<4>?)+>)(?:\\s)*+)?::)*\\s*+)(?:(?:\\s)+)?((?|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.namespace.cuda-cpp', + }, + }, + name: 'meta.body.namespace.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.namespace.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + noexcept_operator: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.noexcept.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.noexcept.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.noexcept.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.noexcept', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + number_literal: { + match: "(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<55>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<55>?)+>)?(?![\\w<:.]))(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:__cdecl|__clrcall|__stdcall|__fastcall|__thiscall|__vectorcall)?)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<55>?)+>)(?:\\s)*+)?::)*+)(operator)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<55>?)+>)(?:\\s)*+)?::)*+)(?:(?:((?:(?:delete\\[\\])|(?:delete)|(?:new\\[\\])|(?:new)|(?:\\->\\*)|(?:<<=)|(?:>>=)|(?:<=>)|(?:\\+\\+)|(?:\\-\\-)|(?:\\(\\))|(?:\\[\\])|(?:\\->)|(?:\\+\\+)|(?:\\-\\-)|(?:<<)|(?:>>)|(?:<=)|(?:>=)|(?:==)|(?:!=)|(?:&&)|(?:\\|\\|)|(?:\\+=)|(?:\\-=)|(?:\\*=)|(?:\\/=)|(?:%=)|(?:&=)|(?:\\^=)|(?:\\|=)|(?:\\+)|(?:\\-)|!|~|(?:\\*)|&|(?:\\*)|(?:\\/)|%|(?:\\+)|(?:\\-)|<|>|&|(?:\\^)|(?:\\|)|=|,))|((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:\\[\\])?)))|("")((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\<|\\()', + end: '(?:(?<=\\}|%>|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.function.definition.special.operator-overload.cuda-cpp', + }, + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '20': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '21': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '22': { + name: 'comment.block.cuda-cpp', + }, + '23': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '24': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '25': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '26': { + name: 'comment.block.cuda-cpp', + }, + '27': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '28': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '29': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '30': { + name: 'comment.block.cuda-cpp', + }, + '31': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '32': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '33': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '34': { + name: 'comment.block.cuda-cpp', + }, + '35': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '36': { + name: 'storage.type.modifier.calling-convention.cuda-cpp', + }, + '37': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '38': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '39': { + name: 'comment.block.cuda-cpp', + }, + '40': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '41': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '42': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '43': { + name: 'comment.block.cuda-cpp', + }, + '44': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '45': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.operator.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'entity.name.operator.type.reference.cuda-cpp', + }, + ], + }, + '59': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '60': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '61': { + name: 'comment.block.cuda-cpp', + }, + '62': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '63': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '64': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '65': { + name: 'comment.block.cuda-cpp', + }, + '66': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '67': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '68': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '69': { + name: 'comment.block.cuda-cpp', + }, + '70': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '71': { + name: 'entity.name.operator.type.array.cuda-cpp', + }, + '72': { + name: 'entity.name.operator.custom-literal.cuda-cpp', + }, + '73': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '74': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '75': { + name: 'comment.block.cuda-cpp', + }, + '76': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '77': { + name: 'entity.name.operator.custom-literal.cuda-cpp', + }, + '78': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '79': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '80': { + name: 'comment.block.cuda-cpp', + }, + '81': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + endCaptures: {}, + name: 'meta.function.definition.special.operator-overload.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.function.definition.special.operator-overload.cuda-cpp', + }, + }, + name: 'meta.head.function.definition.special.operator-overload.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#template_call_range', + }, + { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parameters.begin.bracket.round.special.operator-overload.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parameters.end.bracket.round.special.operator-overload.cuda-cpp', + }, + }, + contentName: 'meta.function.definition.parameters.special.operator-overload', + patterns: [ + { + include: '#function_parameter_context', + }, + { + include: '#evaluation_context', + }, + ], + }, + { + include: '#qualifiers_and_specifiers_post_parameters', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.function.definition.special.operator-overload.cuda-cpp', + }, + }, + name: 'meta.body.function.definition.special.operator-overload.cuda-cpp', + patterns: [ + { + include: '#function_body_context', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.function.definition.special.operator-overload.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + operators: { + patterns: [ + { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.sizeof.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.sizeof.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.sizeof.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.sizeof', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.alignof.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.alignof.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.alignof.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.alignof', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.alignas.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.alignas.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.alignas.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.alignas', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.typeid.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.typeid.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.typeid.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.typeid', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.noexcept.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.noexcept.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.noexcept.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.noexcept', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + begin: '(\\bsizeof\\.\\.\\.)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.sizeof.variadic.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.sizeof.variadic.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.sizeof.variadic.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.sizeof.variadic', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + match: '--', + name: 'keyword.operator.decrement.cuda-cpp', + }, + { + match: '\\+\\+', + name: 'keyword.operator.increment.cuda-cpp', + }, + { + match: '%=|\\+=|-=|\\*=|(?>=|\\|=', + name: 'keyword.operator.assignment.compound.bitwise.cuda-cpp', + }, + { + match: '<<|>>', + name: 'keyword.operator.bitwise.shift.cuda-cpp', + }, + { + match: '!=|<=|>=|==|<|>', + name: 'keyword.operator.comparison.cuda-cpp', + }, + { + match: '&&|!|\\|\\|', + name: 'keyword.operator.logical.cuda-cpp', + }, + { + match: '&|\\||\\^|~', + name: 'keyword.operator.cuda-cpp', + }, + { + include: '#assignment_operator', + }, + { + match: '%|\\*|\\/|-|\\+', + name: 'keyword.operator.cuda-cpp', + }, + { + include: '#ternary_operator', + }, + ], + }, + over_qualified_types: { + patterns: [ + { + match: '(struct)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.struct.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.struct.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + { + match: '(enum)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.enum.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.enum.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + { + match: '(union)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.union.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.union.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + { + match: '(class)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.class.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.class.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + ], + }, + parameter: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\w)', + end: '(?:(?=\\))|(,))', + beginCaptures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + endCaptures: { + '1': { + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + }, + name: 'meta.parameter.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#function_pointer_parameter', + }, + { + include: '#decltype', + }, + { + include: '#vararg_ellipses', + }, + { + match: '((?:((?:(?:__constant__)|(?:__restrict__)|(?:__managed__)|(?:__shared__)|(?:volatile)|(?:register)|(?:restrict)|(?:static)|(?:extern)|(?:const)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))+)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:\\s)*+(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=,|\\)|=)', + captures: { + '1': { + patterns: [ + { + include: '#storage_types', + }, + ], + }, + '2': { + name: 'storage.modifier.specifier.parameter.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + name: 'storage.type.primitive.cuda-cpp storage.type.built-in.primitive.cuda-cpp', + }, + '12': { + name: 'storage.type.cuda-cpp storage.type.built-in.cuda-cpp', + }, + '13': { + name: 'support.type.posix-reserved.pthread.cuda-cpp support.type.built-in.posix-reserved.pthread.cuda-cpp', + }, + '14': { + name: 'support.type.posix-reserved.cuda-cpp support.type.built-in.posix-reserved.cuda-cpp', + }, + '15': { + name: 'entity.name.type.parameter.cuda-cpp', + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + include: '#storage_types', + }, + { + include: '#scope_resolution_parameter_inner_generated', + }, + { + match: '(?:(?:struct)|(?:class)|(?:union)|(?:enum))', + name: 'storage.type.$0.cuda-cpp', + }, + { + begin: '(?<==)', + end: '(?:(?=\\))|(,))', + beginCaptures: {}, + endCaptures: { + '1': { + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + }, + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + match: '\\=', + name: 'keyword.operator.assignment.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\)|,|\\[|=|\\n)', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.parameter.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + include: '#attributes_context', + }, + { + begin: '\\[', + end: '\\]', + beginCaptures: { + '0': { + name: 'punctuation.definition.begin.bracket.square.array.type.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.end.bracket.square.array.type.cuda-cpp', + }, + }, + name: 'meta.bracket.square.array.cuda-cpp', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*))', + captures: { + '0': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '7': { + name: 'comment.block.cuda-cpp', + }, + '8': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + parameter_class: { + match: '(class)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.class.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.class.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + parameter_enum: { + match: '(enum)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.enum.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.enum.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + parameter_or_maybe_value: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\w)', + end: '(?:(?=\\))|(,))', + beginCaptures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + endCaptures: { + '1': { + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + }, + name: 'meta.parameter.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#function_pointer_parameter', + }, + { + include: '#memory_operators', + }, + { + include: '#builtin_storage_type_initilizer', + }, + { + include: '#curly_initializer', + }, + { + include: '#decltype', + }, + { + include: '#vararg_ellipses', + }, + { + match: '((?:((?:(?:__constant__)|(?:__restrict__)|(?:__managed__)|(?:__shared__)|(?:volatile)|(?:register)|(?:restrict)|(?:static)|(?:extern)|(?:const)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))+)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:\\s)*+(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=,|\\)|=)', + captures: { + '1': { + patterns: [ + { + include: '#storage_types', + }, + ], + }, + '2': { + name: 'storage.modifier.specifier.parameter.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + name: 'storage.type.primitive.cuda-cpp storage.type.built-in.primitive.cuda-cpp', + }, + '12': { + name: 'storage.type.cuda-cpp storage.type.built-in.cuda-cpp', + }, + '13': { + name: 'support.type.posix-reserved.pthread.cuda-cpp support.type.built-in.posix-reserved.pthread.cuda-cpp', + }, + '14': { + name: 'support.type.posix-reserved.cuda-cpp support.type.built-in.posix-reserved.cuda-cpp', + }, + '15': { + name: 'entity.name.type.parameter.cuda-cpp', + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + include: '#storage_types', + }, + { + include: '#function_call', + }, + { + include: '#scope_resolution_parameter_inner_generated', + }, + { + match: '(?:(?:struct)|(?:class)|(?:union)|(?:enum))', + name: 'storage.type.$0.cuda-cpp', + }, + { + begin: '(?<==)', + end: '(?:(?=\\))|(,))', + beginCaptures: {}, + endCaptures: { + '1': { + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + }, + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=(?:\\)|,|\\[|=|\\/\\/|(?:(?:\\n)|$)))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'variable.parameter.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + include: '#attributes_context', + }, + { + begin: '\\[', + end: '\\]', + beginCaptures: { + '0': { + name: 'punctuation.definition.begin.bracket.square.array.type.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.end.bracket.square.array.type.cuda-cpp', + }, + }, + name: 'meta.bracket.square.array.cuda-cpp', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*))', + captures: { + '0': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '7': { + name: 'comment.block.cuda-cpp', + }, + '8': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + include: '#evaluation_context', + }, + ], + }, + parameter_struct: { + match: '(struct)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.struct.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.struct.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + parameter_union: { + match: '(union)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:\\[((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\]((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=,|\\)|\\n)', + captures: { + '1': { + name: 'storage.type.union.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.union.parameter.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '15': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '16': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '18': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '19': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '20': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + parentheses: { + begin: '\\(', + end: '\\)', + beginCaptures: { + '0': { + name: 'punctuation.section.parens.begin.bracket.round.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parens.end.bracket.round.cuda-cpp', + }, + }, + name: 'meta.parens.cuda-cpp', + patterns: [ + { + include: '#over_qualified_types', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?pragma\\b', + end: '(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?pragma(?:\\s)+mark)(?:\\s)+(.*)', + captures: { + '1': { + name: 'keyword.control.directive.pragma.pragma-mark.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'punctuation.definition.directive.cuda-cpp', + }, + '5': { + name: 'entity.name.tag.pragma-mark.cuda-cpp', + }, + }, + name: 'meta.preprocessor.pragma.cuda-cpp', + }, + predefined_macros: { + patterns: [ + { + match: '\\b(__cplusplus|__DATE__|__FILE__|__LINE__|__STDC__|__STDC_HOSTED__|__STDC_NO_COMPLEX__|__STDC_VERSION__|__STDCPP_THREADS__|__TIME__|NDEBUG|__OBJC__|__ASSEMBLER__|__ATOM__|__AVX__|__AVX2__|_CHAR_UNSIGNED|__CLR_VER|_CONTROL_FLOW_GUARD|__COUNTER__|__cplusplus_cli|__cplusplus_winrt|_CPPRTTI|_CPPUNWIND|_DEBUG|_DLL|__FUNCDNAME__|__FUNCSIG__|__FUNCTION__|_INTEGRAL_MAX_BITS|__INTELLISENSE__|_ISO_VOLATILE|_KERNEL_MODE|_M_AMD64|_M_ARM|_M_ARM_ARMV7VE|_M_ARM_FP|_M_ARM64|_M_CEE|_M_CEE_PURE|_M_CEE_SAFE|_M_FP_EXCEPT|_M_FP_FAST|_M_FP_PRECISE|_M_FP_STRICT|_M_IX86|_M_IX86_FP|_M_X64|_MANAGED|_MSC_BUILD|_MSC_EXTENSIONS|_MSC_FULL_VER|_MSC_VER|_MSVC_LANG|__MSVC_RUNTIME_CHECKS|_MT|_NATIVE_WCHAR_T_DEFINED|_OPENMP|_PREFAST|__TIMESTAMP__|_VC_NO_DEFAULTLIB|_WCHAR_T_DEFINED|_WIN32|_WIN64|_WINRT_DLL|_ATL_VER|_MFC_VER|__GFORTRAN__|__GNUC__|__GNUC_MINOR__|__GNUC_PATCHLEVEL__|__GNUG__|__STRICT_ANSI__|__BASE_FILE__|__INCLUDE_LEVEL__|__ELF__|__VERSION__|__OPTIMIZE__|__OPTIMIZE_SIZE__|__NO_INLINE__|__GNUC_STDC_INLINE__|__CHAR_UNSIGNED__|__WCHAR_UNSIGNED__|__REGISTER_PREFIX__|__REGISTER_PREFIX__|__SIZE_TYPE__|__PTRDIFF_TYPE__|__WCHAR_TYPE__|__WINT_TYPE__|__INTMAX_TYPE__|__UINTMAX_TYPE__|__SIG_ATOMIC_TYPE__|__INT8_TYPE__|__INT16_TYPE__|__INT32_TYPE__|__INT64_TYPE__|__UINT8_TYPE__|__UINT16_TYPE__|__UINT32_TYPE__|__UINT64_TYPE__|__INT_LEAST8_TYPE__|__INT_LEAST16_TYPE__|__INT_LEAST32_TYPE__|__INT_LEAST64_TYPE__|__UINT_LEAST8_TYPE__|__UINT_LEAST16_TYPE__|__UINT_LEAST32_TYPE__|__UINT_LEAST64_TYPE__|__INT_FAST8_TYPE__|__INT_FAST16_TYPE__|__INT_FAST32_TYPE__|__INT_FAST64_TYPE__|__UINT_FAST8_TYPE__|__UINT_FAST16_TYPE__|__UINT_FAST32_TYPE__|__UINT_FAST64_TYPE__|__INTPTR_TYPE__|__UINTPTR_TYPE__|__CHAR_BIT__|__SCHAR_MAX__|__WCHAR_MAX__|__SHRT_MAX__|__INT_MAX__|__LONG_MAX__|__LONG_LONG_MAX__|__WINT_MAX__|__SIZE_MAX__|__PTRDIFF_MAX__|__INTMAX_MAX__|__UINTMAX_MAX__|__SIG_ATOMIC_MAX__|__INT8_MAX__|__INT16_MAX__|__INT32_MAX__|__INT64_MAX__|__UINT8_MAX__|__UINT16_MAX__|__UINT32_MAX__|__UINT64_MAX__|__INT_LEAST8_MAX__|__INT_LEAST16_MAX__|__INT_LEAST32_MAX__|__INT_LEAST64_MAX__|__UINT_LEAST8_MAX__|__UINT_LEAST16_MAX__|__UINT_LEAST32_MAX__|__UINT_LEAST64_MAX__|__INT_FAST8_MAX__|__INT_FAST16_MAX__|__INT_FAST32_MAX__|__INT_FAST64_MAX__|__UINT_FAST8_MAX__|__UINT_FAST16_MAX__|__UINT_FAST32_MAX__|__UINT_FAST64_MAX__|__INTPTR_MAX__|__UINTPTR_MAX__|__WCHAR_MIN__|__WINT_MIN__|__SIG_ATOMIC_MIN__|__SCHAR_WIDTH__|__SHRT_WIDTH__|__INT_WIDTH__|__LONG_WIDTH__|__LONG_LONG_WIDTH__|__PTRDIFF_WIDTH__|__SIG_ATOMIC_WIDTH__|__SIZE_WIDTH__|__WCHAR_WIDTH__|__WINT_WIDTH__|__INT_LEAST8_WIDTH__|__INT_LEAST16_WIDTH__|__INT_LEAST32_WIDTH__|__INT_LEAST64_WIDTH__|__INT_FAST8_WIDTH__|__INT_FAST16_WIDTH__|__INT_FAST32_WIDTH__|__INT_FAST64_WIDTH__|__INTPTR_WIDTH__|__INTMAX_WIDTH__|__SIZEOF_INT__|__SIZEOF_LONG__|__SIZEOF_LONG_LONG__|__SIZEOF_SHORT__|__SIZEOF_POINTER__|__SIZEOF_FLOAT__|__SIZEOF_DOUBLE__|__SIZEOF_LONG_DOUBLE__|__SIZEOF_SIZE_T__|__SIZEOF_WCHAR_T__|__SIZEOF_WINT_T__|__SIZEOF_PTRDIFF_T__|__BYTE_ORDER__|__ORDER_LITTLE_ENDIAN__|__ORDER_BIG_ENDIAN__|__ORDER_PDP_ENDIAN__|__FLOAT_WORD_ORDER__|__DEPRECATED|__EXCEPTIONS|__GXX_RTTI|__USING_SJLJ_EXCEPTIONS__|__GXX_EXPERIMENTAL_CXX0X__|__GXX_WEAK__|__NEXT_RUNTIME__|__LP64__|_LP64|__SSP__|__SSP_ALL__|__SSP_STRONG__|__SSP_EXPLICIT__|__SANITIZE_ADDRESS__|__SANITIZE_THREAD__|__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1|__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2|__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4|__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8|__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16|__HAVE_SPECULATION_SAFE_VALUE|__GCC_HAVE_DWARF2_CFI_ASM|__FP_FAST_FMA|__FP_FAST_FMAF|__FP_FAST_FMAL|__FP_FAST_FMAF16|__FP_FAST_FMAF32|__FP_FAST_FMAF64|__FP_FAST_FMAF128|__FP_FAST_FMAF32X|__FP_FAST_FMAF64X|__FP_FAST_FMAF128X|__GCC_IEC_559|__GCC_IEC_559_COMPLEX|__NO_MATH_ERRNO__|__has_builtin|__has_feature|__has_extension|__has_cpp_attribute|__has_c_attribute|__has_attribute|__has_declspec_attribute|__is_identifier|__has_include|__has_include_next|__has_warning|__BASE_FILE__|__FILE_NAME__|__clang__|__clang_major__|__clang_minor__|__clang_patchlevel__|__clang_version__|__fp16|_Float16)\\b', + captures: { + '1': { + name: 'entity.name.other.preprocessor.macro.predefined.$1.cuda-cpp', + }, + }, + }, + { + match: '\\b__([A-Z_]+)__\\b', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$1.cuda-cpp', + }, + ], + }, + preprocessor_conditional_context: { + patterns: [ + { + include: '#preprocessor_conditional_defined', + }, + { + include: '#comments', + }, + { + include: '#language_constants', + }, + { + include: '#string_context', + }, + { + include: '#d9bc4796b0b_preprocessor_number_literal', + }, + { + include: '#operators', + }, + { + include: '#predefined_macros', + }, + { + include: '#macro_name', + }, + { + include: '#line_continuation_character', + }, + ], + }, + preprocessor_conditional_defined: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?((?:(?:ifndef|ifdef)|if))', + end: '^(?!\\s*+#\\s*(?:else|endif))', + beginCaptures: { + '0': { + name: 'keyword.control.directive.conditional.$6.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'punctuation.definition.directive.cuda-cpp', + }, + '6': {}, + }, + endCaptures: {}, + patterns: [ + { + begin: '\\G(?<=ifndef|ifdef|if)', + end: '(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'punctuation.definition.directive.cuda-cpp', + }, + }, + name: 'keyword.control.directive.$4.cuda-cpp', + }, + preprocessor_context: { + patterns: [ + { + include: '#pragma_mark', + }, + { + include: '#pragma', + }, + { + include: '#include', + }, + { + include: '#line', + }, + { + include: '#diagnostic', + }, + { + include: '#undef', + }, + { + include: '#preprocessor_conditional_range', + }, + { + include: '#single_line_macro', + }, + { + include: '#macro', + }, + { + include: '#preprocessor_conditional_standalone', + }, + { + include: '#macro_argument', + }, + ], + }, + qualified_type: { + match: '\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<11>?)+>)(?:\\s)*+)?::)*+)?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<11>?)+>)?(?![\\w<:.])', + captures: { + '0': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '1': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '5': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '6': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + name: 'meta.qualified_type.cuda-cpp', + }, + qualifiers_and_specifiers_post_parameters: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?:((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'storage.modifier.specifier.functional.post-parameters.$3.cuda-cpp', + }, + '4': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '5': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + scope_resolution: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_function_call: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_function_call_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.call.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_function_call_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_function_call_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.call.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.function.call.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.call.cuda-cpp', + }, + }, + }, + scope_resolution_function_definition: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_function_definition_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_function_definition_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_function_definition_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.function.definition.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.cuda-cpp', + }, + }, + }, + scope_resolution_function_definition_operator_overload: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_function_definition_operator_overload_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.operator-overload.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_function_definition_operator_overload_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_function_definition_operator_overload_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.operator-overload.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.function.definition.operator-overload.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.function.definition.operator-overload.cuda-cpp', + }, + }, + }, + scope_resolution_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + }, + }, + scope_resolution_namespace_alias: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_namespace_alias_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.alias.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_namespace_alias_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_namespace_alias_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.alias.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.namespace.alias.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.alias.cuda-cpp', + }, + }, + }, + scope_resolution_namespace_block: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_namespace_block_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.block.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_namespace_block_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_namespace_block_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.block.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.namespace.block.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.block.cuda-cpp', + }, + }, + }, + scope_resolution_namespace_using: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_namespace_using_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.using.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_namespace_using_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_namespace_using_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.using.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.namespace.using.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.namespace.using.cuda-cpp', + }, + }, + }, + scope_resolution_parameter: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_parameter_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.parameter.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_parameter_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_parameter_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.parameter.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.parameter.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.parameter.cuda-cpp', + }, + }, + }, + scope_resolution_template_call: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_template_call_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.template.call.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_template_call_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_template_call_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.template.call.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.template.call.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.template.call.cuda-cpp', + }, + }, + }, + scope_resolution_template_definition: { + match: '(::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<3>?)+>)(?:\\s)*+)?::)*\\s*+', + captures: { + '0': { + patterns: [ + { + include: '#scope_resolution_template_definition_inner_generated', + }, + ], + }, + '1': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.template.definition.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + }, + scope_resolution_template_definition_inner_generated: { + match: '((::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?::)*\\s*+)((?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<7>?)+>)(?:\\s)*+)?(::)', + captures: { + '1': { + patterns: [ + { + include: '#scope_resolution_template_definition_inner_generated', + }, + ], + }, + '2': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.template.definition.cuda-cpp', + }, + '3': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '4': {}, + '5': { + name: 'entity.name.scope-resolution.template.definition.cuda-cpp', + }, + '6': { + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + '7': {}, + '8': { + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.template.definition.cuda-cpp', + }, + }, + }, + semicolon: { + match: ';', + name: 'punctuation.terminator.statement.cuda-cpp', + }, + simple_type: { + match: '(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)(?:\\s)*+)?::)*+)?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<12>?)+>)?(?![\\w<:.]))(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?', + captures: { + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': {}, + '13': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '14': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '15': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + single_line_macro: { + match: '^((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))#define.*(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + sizeof_operator: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.sizeof.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.sizeof.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.sizeof.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.sizeof', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + sizeof_variadic_operator: { + begin: '(\\bsizeof\\.\\.\\.)((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.sizeof.variadic.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.sizeof.variadic.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.sizeof.variadic.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.sizeof.variadic', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + square_brackets: { + name: 'meta.bracket.square.access', + begin: '([a-zA-Z_][a-zA-Z_0-9]*|(?<=[\\]\\)]))?(\\[)(?!\\])', + beginCaptures: { + '1': { + name: 'variable.other.object', + }, + '2': { + name: 'punctuation.definition.begin.bracket.square', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'punctuation.definition.end.bracket.square', + }, + }, + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + standard_declares: { + patterns: [ + { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.struct.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.struct.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.union.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.union.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.enum.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.enum.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.class.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.class.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + ], + }, + static_assert: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'keyword.other.static_assert.cuda-cpp', + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + name: 'punctuation.section.arguments.begin.bracket.round.static_assert.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.static_assert.cuda-cpp', + }, + }, + patterns: [ + { + begin: '(,)(?:(?:\\s)+)?(?=(?:L|u8|u|U(?:(?:\\s)+)?\\")?)', + end: '(?=\\))', + beginCaptures: { + '1': { + name: 'punctuation.separator.delimiter.comma.cuda-cpp', + }, + }, + endCaptures: {}, + name: 'meta.static_assert.message.cuda-cpp', + patterns: [ + { + include: '#string_context', + }, + ], + }, + { + include: '#evaluation_context', + }, + ], + }, + std_space: { + match: '(?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))', + captures: { + '0': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '1': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + storage_specifiers: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'storage.modifier.specifier.$3.cuda-cpp', + }, + }, + }, + storage_types: { + patterns: [ + { + include: '#storage_specifiers', + }, + { + include: '#inline_builtin_storage_type', + }, + { + include: '#decltype', + }, + { + include: '#typename', + }, + ], + }, + string_context: { + patterns: [ + { + begin: '((?:u|u8|U|L)?)"', + end: '"', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.cuda-cpp', + }, + '1': { + name: 'meta.encoding.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.string.end.cuda-cpp', + }, + }, + name: 'string.quoted.double.cuda-cpp', + patterns: [ + { + match: '(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8})', + name: 'constant.character.escape.cuda-cpp', + }, + { + match: '\\\\[\'"?\\\\abfnrtv]', + name: 'constant.character.escape.cuda-cpp', + }, + { + match: '\\\\[0-7]{1,3}', + name: 'constant.character.escape.cuda-cpp', + }, + { + match: '(?:(\\\\x0*[0-9a-fA-F]{2}(?![0-9a-fA-F]))|((?:\\\\x[0-9a-fA-F]*|\\\\x)))', + captures: { + '1': { + name: 'constant.character.escape.cuda-cpp', + }, + '2': { + name: 'invalid.illegal.unknown-escape.cuda-cpp', + }, + }, + }, + { + include: '#string_escapes_context_c', + }, + ], + }, + { + begin: "(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?={)|(?:((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:(?!:)))?)', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.struct.cuda-cpp', + }, + '1': { + name: 'storage.type.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=:|{|$)', + captures: { + '1': { + name: 'entity.name.type.struct.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: 'DLLEXPORT', + name: 'entity.name.other.preprocessor.macro.predefined.DLLEXPORT.cuda-cpp', + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$0.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '20': { + name: 'punctuation.separator.colon.inheritance.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.struct.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.struct.cuda-cpp', + }, + }, + name: 'meta.head.struct.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#inheritance_context', + }, + { + include: '#template_call_range', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.struct.cuda-cpp', + }, + }, + name: 'meta.body.struct.cuda-cpp', + patterns: [ + { + include: '#function_pointer', + }, + { + include: '#static_assert', + }, + { + include: '#constructor_inline', + }, + { + include: '#destructor_inline', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.struct.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + struct_declare: { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.struct.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.struct.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + switch_conditional_parentheses: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'punctuation.section.parens.begin.bracket.round.conditional.switch.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.parens.end.bracket.round.conditional.switch.cuda-cpp', + }, + }, + name: 'meta.conditional.switch.cuda-cpp', + patterns: [ + { + include: '#evaluation_context', + }, + { + include: '#c_conditional_context', + }, + ], + }, + switch_statement: { + begin: '((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?|\\?\\?>)|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.switch.cuda-cpp', + }, + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '5': { + name: 'keyword.control.switch.cuda-cpp', + }, + }, + endCaptures: {}, + name: 'meta.block.switch.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.switch.cuda-cpp', + }, + }, + name: 'meta.head.switch.cuda-cpp', + patterns: [ + { + include: '#switch_conditional_parentheses', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.switch.cuda-cpp', + }, + }, + name: 'meta.body.switch.cuda-cpp', + patterns: [ + { + include: '#default_statement', + }, + { + include: '#case_statement', + }, + { + include: '$self', + }, + { + include: '#block_innards', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.switch.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + template_argument_defaulted: { + match: '(?<=<|,)(?:(?:\\s)+)?((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*(?:\\s)+)*)((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)(?:(?:\\s)+)?([=])', + captures: { + '1': { + name: 'storage.type.template.cuda-cpp', + }, + '2': { + name: 'entity.name.type.template.cuda-cpp', + }, + '3': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + }, + }, + template_call_context: { + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#template_call_range', + }, + { + include: '#storage_types', + }, + { + include: '#language_constants', + }, + { + include: '#scope_resolution_template_call_inner_generated', + }, + { + include: '#operators', + }, + { + include: '#number_literal', + }, + { + include: '#string_context', + }, + { + include: '#comma_in_template_argument', + }, + { + include: '#qualified_type', + }, + ], + }, + template_call_innards: { + match: '((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<1>?)+>)(?:\\s)*+', + captures: { + '0': { + patterns: [ + { + include: '#template_call_range', + }, + ], + }, + }, + name: 'meta.template.call.cuda-cpp', + }, + template_call_range: { + begin: '<', + end: '>', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + template_definition: { + begin: '(?', + beginCaptures: { + '1': { + name: 'storage.type.template.cuda-cpp', + }, + '2': { + name: 'punctuation.section.angle-brackets.start.template.definition.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.definition.cuda-cpp', + }, + }, + name: 'meta.template.definition.cuda-cpp', + patterns: [ + { + begin: '(?<=\\w)(?:(?:\\s)+)?<', + end: '>', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + include: '#template_definition_context', + }, + ], + }, + template_definition_argument: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)|((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*(?:\\s)+)+)((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*))|((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)(?:(?:\\s)+)?(\\.\\.\\.)(?:(?:\\s)+)?((?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*))(?:(?:\\s)+)?(?:(,)|(?=>|$))', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'storage.type.template.argument.$3.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'storage.type.template.argument.$0.cuda-cpp', + }, + ], + }, + '5': { + name: 'entity.name.type.template.cuda-cpp', + }, + '6': { + name: 'storage.type.template.cuda-cpp', + }, + '7': { + name: 'punctuation.vararg-ellipses.template.definition.cuda-cpp', + }, + '8': { + name: 'entity.name.type.template.cuda-cpp', + }, + '9': { + name: 'punctuation.separator.delimiter.comma.template.argument.cuda-cpp', + }, + }, + }, + template_definition_context: { + patterns: [ + { + include: '#scope_resolution_template_definition_inner_generated', + }, + { + include: '#template_definition_argument', + }, + { + include: '#template_argument_defaulted', + }, + { + include: '#template_call_innards', + }, + { + include: '#evaluation_context', + }, + ], + }, + template_isolated_definition: { + match: '(?(?:(?:\\s)+)?$)', + captures: { + '1': { + name: 'storage.type.template.cuda-cpp', + }, + '2': { + name: 'punctuation.section.angle-brackets.start.template.definition.cuda-cpp', + }, + '3': { + name: 'meta.template.definition.cuda-cpp', + patterns: [ + { + include: '#template_definition_context', + }, + ], + }, + '4': { + name: 'punctuation.section.angle-brackets.end.template.definition.cuda-cpp', + }, + }, + }, + ternary_operator: { + begin: '\\?', + end: ':', + beginCaptures: { + '0': { + name: 'keyword.operator.ternary.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'keyword.operator.ternary.cuda-cpp', + }, + }, + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#string_context', + }, + { + include: '#number_literal', + }, + { + include: '#method_access', + }, + { + include: '#member_access', + }, + { + include: '#predefined_macros', + }, + { + include: '#operators', + }, + { + include: '#memory_operators', + }, + { + include: '#wordlike_operators', + }, + { + include: '#type_casting_operators', + }, + { + include: '#control_flow_keywords', + }, + { + include: '#exception_keywords', + }, + { + include: '#the_this_keyword', + }, + { + include: '#language_constants', + }, + { + include: '#builtin_storage_type_initilizer', + }, + { + include: '#qualifiers_and_specifiers_post_parameters', + }, + { + include: '#functional_specifiers_pre_parameters', + }, + { + include: '#storage_types', + }, + { + include: '#lambdas', + }, + { + include: '#attributes_context', + }, + { + include: '#parentheses', + }, + { + include: '#function_call', + }, + { + include: '#scope_resolution_inner_generated', + }, + { + include: '#square_brackets', + }, + { + include: '#semicolon', + }, + { + include: '#comma', + }, + ], + applyEndPatternLast: true, + }, + the_this_keyword: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'variable.language.this.cuda-cpp', + }, + }, + }, + type_alias: { + match: '(using)(?:(?:\\s)+)?(?!namespace)(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<29>?)+>)(?:\\s)*+)?::)*+)?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<29>?)+>)?(?![\\w<:.]))(?:(?:\\s)+)?(\\=)(?:(?:\\s)+)?((?:typename)?)(?:(?:\\s)+)?((?:(?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<29>?)+>)(?:\\s)*+)?::)*+)?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<29>?)+>)?(?![\\w<:.]))|(.*(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?:(\\[)(\\w*)(\\])(?:(?:\\s)+)?)?(?:(?:\\s)+)?(?:(;)|\\n)', + captures: { + '1': { + name: 'keyword.other.using.directive.cuda-cpp', + }, + '2': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '3': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '4': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '5': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '14': { + name: 'keyword.operator.assignment.cuda-cpp', + }, + '15': { + name: 'keyword.other.typename.cuda-cpp', + }, + '16': { + patterns: [ + { + include: '#storage_specifiers', + }, + ], + }, + '17': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '18': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '19': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '20': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '21': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '22': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '23': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '24': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '30': { + name: 'meta.declaration.type.alias.value.unknown.cuda-cpp', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + '31': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '32': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '33': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '34': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '35': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '36': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '37': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '38': { + name: 'punctuation.definition.begin.bracket.square.cuda-cpp', + }, + '39': { + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + '40': { + name: 'punctuation.definition.end.bracket.square.cuda-cpp', + }, + '41': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.declaration.type.alias.cuda-cpp', + }, + type_casting_operators: { + match: '((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '3': { + name: 'keyword.operator.wordlike.cuda-cpp keyword.operator.cast.$3.cuda-cpp', + }, + }, + }, + typedef_class: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?={)|(?:((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:(?!:)))?)', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.class.cuda-cpp', + }, + '1': { + name: 'storage.type.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=:|{|$)', + captures: { + '1': { + name: 'entity.name.type.class.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: 'DLLEXPORT', + name: 'entity.name.other.preprocessor.macro.predefined.DLLEXPORT.cuda-cpp', + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$0.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '20': { + name: 'punctuation.separator.colon.inheritance.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.class.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.class.cuda-cpp', + }, + }, + name: 'meta.head.class.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#inheritance_context', + }, + { + include: '#template_call_range', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.class.cuda-cpp', + }, + }, + name: 'meta.body.class.cuda-cpp', + patterns: [ + { + include: '#function_pointer', + }, + { + include: '#static_assert', + }, + { + include: '#constructor_inline', + }, + { + include: '#destructor_inline', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.class.cuda-cpp', + patterns: [ + { + match: '(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '12': { + name: 'comment.block.cuda-cpp', + }, + '13': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '14': { + name: 'entity.name.type.alias.cuda-cpp', + }, + }, + }, + { + match: ',', + }, + ], + }, + ], + }, + ], + }, + typedef_function_pointer: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)(?:\\s)*+)?::)*+)?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<18>?)+>)?(?![\\w<:.]))(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()(\\*)(?:(?:\\s)+)?((?:(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)?)(?:(?:\\s)+)?(?:(\\[)(\\w*)(\\])(?:(?:\\s)+)?)*(\\))(?:(?:\\s)+)?(\\()', + end: '(\\))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=[{=,);>]|\\n)(?!\\()', + beginCaptures: { + '1': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '3': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '4': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '5': { + name: 'comment.block.cuda-cpp', + }, + '6': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '20': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '21': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '22': { + name: 'comment.block.cuda-cpp', + }, + '23': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '24': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '25': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '26': { + name: 'comment.block.cuda-cpp', + }, + '27': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '28': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '29': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '30': { + name: 'comment.block.cuda-cpp', + }, + '31': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '32': { + name: 'punctuation.section.parens.begin.bracket.round.function.pointer.cuda-cpp', + }, + '33': { + name: 'punctuation.definition.function.pointer.dereference.cuda-cpp', + }, + '34': { + name: 'entity.name.type.alias.cuda-cpp entity.name.type.pointer.function.cuda-cpp', + }, + '35': { + name: 'punctuation.definition.begin.bracket.square.cuda-cpp', + }, + '36': { + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + '37': { + name: 'punctuation.definition.end.bracket.square.cuda-cpp', + }, + '38': { + name: 'punctuation.section.parens.end.bracket.round.function.pointer.cuda-cpp', + }, + '39': { + name: 'punctuation.section.parameters.begin.bracket.round.function.pointer.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.section.parameters.end.bracket.round.function.pointer.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + patterns: [ + { + include: '#function_parameter_context', + }, + ], + }, + ], + }, + typedef_struct: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?={)|(?:((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:(?!:)))?)', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.struct.cuda-cpp', + }, + '1': { + name: 'storage.type.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=:|{|$)', + captures: { + '1': { + name: 'entity.name.type.struct.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: 'DLLEXPORT', + name: 'entity.name.other.preprocessor.macro.predefined.DLLEXPORT.cuda-cpp', + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$0.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '20': { + name: 'punctuation.separator.colon.inheritance.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.struct.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.struct.cuda-cpp', + }, + }, + name: 'meta.head.struct.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#inheritance_context', + }, + { + include: '#template_call_range', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.struct.cuda-cpp', + }, + }, + name: 'meta.body.struct.cuda-cpp', + patterns: [ + { + include: '#function_pointer', + }, + { + include: '#static_assert', + }, + { + include: '#constructor_inline', + }, + { + include: '#destructor_inline', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.struct.cuda-cpp', + patterns: [ + { + match: '(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '12': { + name: 'comment.block.cuda-cpp', + }, + '13': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '14': { + name: 'entity.name.type.alias.cuda-cpp', + }, + }, + }, + { + match: ',', + }, + ], + }, + ], + }, + ], + }, + typedef_union: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?={)|(?:((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:(?!:)))?)', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.union.cuda-cpp', + }, + '1': { + name: 'storage.type.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=:|{|$)', + captures: { + '1': { + name: 'entity.name.type.union.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: 'DLLEXPORT', + name: 'entity.name.other.preprocessor.macro.predefined.DLLEXPORT.cuda-cpp', + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$0.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '20': { + name: 'punctuation.separator.colon.inheritance.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.union.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.union.cuda-cpp', + }, + }, + name: 'meta.head.union.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#inheritance_context', + }, + { + include: '#template_call_range', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.union.cuda-cpp', + }, + }, + name: 'meta.body.union.cuda-cpp', + patterns: [ + { + include: '#function_pointer', + }, + { + include: '#static_assert', + }, + { + include: '#constructor_inline', + }, + { + include: '#destructor_inline', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.union.cuda-cpp', + patterns: [ + { + match: '(((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '8': { + name: 'comment.block.cuda-cpp', + }, + '9': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '12': { + name: 'comment.block.cuda-cpp', + }, + '13': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '14': { + name: 'entity.name.type.alias.cuda-cpp', + }, + }, + }, + { + match: ',', + }, + ], + }, + ], + }, + ], + }, + typeid_operator: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\()', + end: '\\)', + beginCaptures: { + '1': { + name: 'keyword.operator.functionlike.cuda-cpp keyword.operator.typeid.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'punctuation.section.arguments.begin.bracket.round.operator.typeid.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.arguments.end.bracket.round.operator.typeid.cuda-cpp', + }, + }, + contentName: 'meta.arguments.operator.typeid', + patterns: [ + { + include: '#evaluation_context', + }, + ], + }, + typename: { + match: '(((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(\\s*+((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?:(?:(?:unsigned)|(?:signed)|(?:short)|(?:long))|(?:(?:struct)|(?:class)|(?:union)|(?:enum)))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*((?:::)?(?:(?!\\b(?:__has_cpp_attribute|reinterpret_cast|__forceinline__|atomic_noexcept|__has_include|atomic_cancel|atomic_commit|dynamic_cast|__constant__|__restrict__|__noinline__|thread_local|synchronized|static_cast|__managed__|const_cast|__shared__|__global__|__device__|co_return|constexpr|constexpr|constexpr|consteval|protected|namespace|co_return|noexcept|noexcept|continue|co_await|co_yield|volatile|register|restrict|explicit|__host__|volatile|noexcept|template|operator|decltype|typename|requires|co_await|co_yield|reflexpr|alignof|alignas|default|mutable|virtual|mutable|private|include|warning|_Pragma|defined|typedef|__asm__|concept|sizeof|delete|not_eq|bitand|and_eq|xor_eq|typeid|switch|return|struct|static|extern|inline|friend|public|ifndef|define|pragma|export|import|module|compl|bitor|throw|or_eq|while|catch|break|class|union|const|const|endif|ifdef|undef|error|using|else|goto|case|enum|elif|else|line|this|not|new|xor|and|for|try|asm|or|do|if|if)\\b)(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<17>?)+>)(?:\\s)*+)?::)*+)?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?!(?:(?:transaction_safe_dynamic)|(?:__has_cpp_attribute)|(?:reinterpret_cast)|(?:transaction_safe)|(?:__forceinline__)|(?:atomic_noexcept)|(?:__has_include)|(?:atomic_cancel)|(?:atomic_commit)|(?:dynamic_cast)|(?:__constant__)|(?:__restrict__)|(?:__noinline__)|(?:thread_local)|(?:synchronized)|(?:static_cast)|(?:__managed__)|(?:const_cast)|(?:__shared__)|(?:__global__)|(?:__device__)|(?:co_return)|(?:constexpr)|(?:constexpr)|(?:constexpr)|(?:consteval)|(?:protected)|(?:threadIdx)|(?:namespace)|(?:co_return)|(?:noexcept)|(?:noexcept)|(?:continue)|(?:co_await)|(?:co_yield)|(?:volatile)|(?:register)|(?:restrict)|(?:explicit)|(?:__host__)|(?:override)|(?:volatile)|(?:noexcept)|(?:blockIdx)|(?:blockDim)|(?:warpSize)|(?:template)|(?:operator)|(?:decltype)|(?:typename)|(?:requires)|(?:co_await)|(?:co_yield)|(?:reflexpr)|(?:alignof)|(?:alignas)|(?:default)|(?:nullptr)|(?:mutable)|(?:virtual)|(?:mutable)|(?:private)|(?:include)|(?:warning)|(?:_Pragma)|(?:defined)|(?:gridDim)|(?:typedef)|(?:__asm__)|(?:concept)|(?:sizeof)|(?:delete)|(?:not_eq)|(?:bitand)|(?:and_eq)|(?:xor_eq)|(?:typeid)|(?:switch)|(?:return)|(?:static)|(?:extern)|(?:inline)|(?:friend)|(?:public)|(?:ifndef)|(?:define)|(?:pragma)|(?:export)|(?:import)|(?:module)|(?:compl)|(?:bitor)|(?:throw)|(?:or_eq)|(?:while)|(?:catch)|(?:break)|(?:false)|(?:const)|(?:final)|(?:const)|(?:endif)|(?:ifdef)|(?:undef)|(?:error)|(?:using)|(?:audit)|(?:axiom)|(?:else)|(?:goto)|(?:case)|(?:NULL)|(?:true)|(?:elif)|(?:else)|(?:line)|(?:this)|(?:not)|(?:new)|(?:xor)|(?:and)|(?:for)|(?:try)|(?:asm)|(?:or)|(?:do)|(?:if)|(?:if))\\b)(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*\\b((?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<17>?)+>)?(?![\\w<:.]))', + captures: { + '1': { + name: 'storage.modifier.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '5': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '6': { + name: 'meta.qualified_type.cuda-cpp', + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.cuda-cpp', + }, + { + match: '(?', + beginCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.begin.template.call.cuda-cpp', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.angle-brackets.end.template.call.cuda-cpp', + }, + }, + name: 'meta.template.call.cuda-cpp', + patterns: [ + { + include: '#template_call_context', + }, + ], + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.type.cuda-cpp', + }, + ], + }, + '7': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + patterns: [ + { + match: '::', + name: 'punctuation.separator.namespace.access.cuda-cpp punctuation.separator.scope-resolution.type.cuda-cpp', + }, + { + match: '(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '17': {}, + }, + }, + undef: { + match: '(^((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(#)(?:(?:\\s)+)?undef\\b)((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'punctuation.definition.directive.cuda-cpp', + }, + '5': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '6': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '7': { + name: 'entity.name.function.preprocessor.cuda-cpp', + }, + }, + name: 'meta.preprocessor.undef.cuda-cpp', + }, + union_block: { + begin: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:(?={)|(?:((?:(?:(?:\\[\\[.*?\\]\\]|__attribute(?:__)?\\s*\\(\\s*\\(.*?\\)\\s*\\))|__declspec\\(.*?\\))|alignas\\(.*?\\))(?!\\)))((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?((?:(?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*+)?(?:((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(:(?!:)))?)', + end: '(?:(?:(?<=\\}|%>|\\?\\?>)(?:(?:\\s)+)?(;)|(;))|(?=[;>\\[\\]=]))', + beginCaptures: { + '0': { + name: 'meta.head.union.cuda-cpp', + }, + '1': { + name: 'storage.type.$1.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#attributes_context', + }, + { + include: '#number_literal', + }, + ], + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '11': { + patterns: [ + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))', + captures: { + '1': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: '((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?:((?(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))?(?=:|{|$)', + captures: { + '1': { + name: 'entity.name.type.union.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '4': { + name: 'comment.block.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '6': { + name: 'storage.type.modifier.final.cuda-cpp', + }, + '7': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '8': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '9': { + name: 'comment.block.cuda-cpp', + }, + '10': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + { + match: 'DLLEXPORT', + name: 'entity.name.other.preprocessor.macro.predefined.DLLEXPORT.cuda-cpp', + }, + { + match: '(?:[a-zA-Z_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*', + name: 'entity.name.other.preprocessor.macro.predefined.probably.$0.cuda-cpp', + }, + ], + }, + '12': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '13': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '14': { + name: 'comment.block.cuda-cpp', + }, + '15': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '16': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '17': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '18': { + name: 'comment.block.cuda-cpp', + }, + '19': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + '20': { + name: 'punctuation.separator.colon.inheritance.cuda-cpp', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + '2': { + name: 'punctuation.terminator.statement.cuda-cpp', + }, + }, + name: 'meta.block.union.cuda-cpp', + patterns: [ + { + begin: '\\G ?', + end: '(?:\\{|<%|\\?\\?<|(?=;))', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.begin.bracket.curly.union.cuda-cpp', + }, + }, + name: 'meta.head.union.cuda-cpp', + patterns: [ + { + include: '#ever_present_context', + }, + { + include: '#inheritance_context', + }, + { + include: '#template_call_range', + }, + ], + }, + { + begin: '(?<=\\{|<%|\\?\\?<)', + end: '\\}|%>|\\?\\?>', + beginCaptures: {}, + endCaptures: { + '0': { + name: 'punctuation.section.block.end.bracket.curly.union.cuda-cpp', + }, + }, + name: 'meta.body.union.cuda-cpp', + patterns: [ + { + include: '#function_pointer', + }, + { + include: '#static_assert', + }, + { + include: '#constructor_inline', + }, + { + include: '#destructor_inline', + }, + { + include: '$self', + }, + ], + }, + { + begin: '(?<=\\}|%>|\\?\\?>)[\\s]*', + end: '[\\s]*(?=;)', + beginCaptures: {}, + endCaptures: {}, + name: 'meta.tail.union.cuda-cpp', + patterns: [ + { + include: '$self', + }, + ], + }, + ], + }, + union_declare: { + match: '((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))?(?:(?:&|(?:\\*))((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z))))*(?:&|(?:\\*)))?((?:((?:(?>(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))\\b(?!override\\W|override\\$|final\\W|final\\$)((?(?:\\s)+)|\\/\\*(?:[^\\*]|(?:\\*)++[^\\/])*+(?:\\*)++\\/)+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))(?=\\S)(?![:{a-zA-Z])', + captures: { + '1': { + name: 'storage.type.union.declare.cuda-cpp', + }, + '2': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '3': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '4': { + name: 'entity.name.type.union.cuda-cpp', + }, + '5': { + patterns: [ + { + match: '\\*', + name: 'storage.modifier.pointer.cuda-cpp', + }, + { + match: '(?:\\&((?:(?:(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))+)|(?:\\b)|(?=\\W)|(?<=\\W)|(?:\\A)|(?:\\Z)))){2,}\\&', + captures: { + '1': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '2': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '3': { + name: 'comment.block.cuda-cpp', + }, + '4': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + name: 'invalid.illegal.reference-type.cuda-cpp', + }, + { + match: '\\&', + name: 'storage.modifier.reference.cuda-cpp', + }, + ], + }, + '6': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '7': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '8': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '9': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '10': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '11': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + '12': { + name: 'variable.other.object.declare.cuda-cpp', + }, + '13': { + patterns: [ + { + include: '#inline_comment', + }, + ], + }, + '14': { + patterns: [ + { + match: '(?:(?>(?:\\s)+)|(\\/\\*)((?:[^\\*]|(?:\\*)++[^\\/])*+((?:\\*)++\\/)))', + captures: { + '1': { + name: 'comment.block.cuda-cpp punctuation.definition.comment.begin.cuda-cpp', + }, + '2': { + name: 'comment.block.cuda-cpp', + }, + '3': { + patterns: [ + { + match: '\\*\\/', + name: 'comment.block.cuda-cpp punctuation.definition.comment.end.cuda-cpp', + }, + { + match: '\\*', + name: 'comment.block.cuda-cpp', + }, + ], + }, + }, + }, + ], + }, + }, + }, + using_name: { + match: '(using)(?:\\s)+(?!namespace\\b)', + captures: { + '1': { + name: 'keyword.other.using.directive.cuda-cpp', + }, + }, + }, + using_namespace: { + begin: '(?]*+|"(?:[^"]*|\\\\")")|\'(?:[^\']*|\\\\\')\')\\g<6>?)+>)(?:\\s)*+)?::)*\\s*+)?((?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + beginCaptures: { + '1': { + name: 'meta.definition.variable.js.jsx entity.name.function.js.jsx', + }, + '2': { + name: 'keyword.operator.definiteassignment.js.jsx', + }, + }, + end: '(?=$|^|[;,=}]|((?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + beginCaptures: { + '1': { + name: 'meta.definition.variable.js.jsx variable.other.constant.js.jsx entity.name.function.js.jsx', + }, + }, + end: '(?=$|^|[;,=}]|((?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '1': { + name: 'storage.modifier.js.jsx', + }, + '2': { + name: 'keyword.operator.rest.js.jsx', + }, + '3': { + name: 'entity.name.function.js.jsx variable.language.this.js.jsx', + }, + '4': { + name: 'entity.name.function.js.jsx', + }, + '5': { + name: 'keyword.operator.optional.js.jsx', + }, + }, + }, + { + match: '(?x)(?:(?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '1': { + name: 'meta.definition.property.js.jsx entity.name.function.js.jsx', + }, + '2': { + name: 'keyword.operator.optional.js.jsx', + }, + '3': { + name: 'keyword.operator.definiteassignment.js.jsx', + }, + }, + }, + { + name: 'meta.definition.property.js.jsx variable.object.property.js.jsx', + match: '\\#?[_$[:alpha:]][_$[:alnum:]]*', + }, + { + name: 'keyword.operator.optional.js.jsx', + match: '\\?', + }, + { + name: 'keyword.operator.definiteassignment.js.jsx', + match: '\\!', + }, + ], + }, + 'variable-initializer': { + patterns: [ + { + begin: '(?\\s*$)', + beginCaptures: { + '1': { + name: 'keyword.operator.assignment.js.jsx', + }, + }, + end: '(?=$|^|[,);}\\]]|((?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.js.jsx', + }, + '2': { + name: 'storage.modifier.js.jsx', + }, + '3': { + name: 'storage.modifier.js.jsx', + }, + '4': { + name: 'storage.modifier.async.js.jsx', + }, + '5': { + name: 'keyword.operator.new.js.jsx', + }, + '6': { + name: 'keyword.generator.asterisk.js.jsx', + }, + }, + end: '(?=\\}|;|,|$)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + ], + }, + { + name: 'meta.method.declaration.js.jsx', + begin: '(?x)(?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.js.jsx', + }, + '2': { + name: 'storage.modifier.js.jsx', + }, + '3': { + name: 'storage.modifier.js.jsx', + }, + '4': { + name: 'storage.modifier.async.js.jsx', + }, + '5': { + name: 'storage.type.property.js.jsx', + }, + '6': { + name: 'keyword.generator.asterisk.js.jsx', + }, + }, + end: '(?=\\}|;|,|$)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + ], + }, + ], + }, + 'object-literal-method-declaration': { + name: 'meta.method.declaration.js.jsx', + begin: '(?x)(?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + '2': { + name: 'storage.type.property.js.jsx', + }, + '3': { + name: 'keyword.generator.asterisk.js.jsx', + }, + }, + end: '(?=\\}|;|,)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + { + begin: '(?x)(?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + '2': { + name: 'storage.type.property.js.jsx', + }, + '3': { + name: 'keyword.generator.asterisk.js.jsx', + }, + }, + end: '(?=\\(|\\<)', + patterns: [ + { + include: '#method-declaration-name', + }, + ], + }, + ], + }, + 'method-declaration-name': { + begin: '(?x)(?=((\\b(?)', + captures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + '2': { + name: 'variable.parameter.js.jsx', + }, + }, + }, + { + name: 'meta.arrow.js.jsx', + begin: '(?x) (?:\n (? is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n )\n)', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + }, + end: '(?==>|\\{|(^\\s*(export|function|class|interface|let|var|const|import|enum|namespace|module|type|abstract|declare)\\s+))', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-parameters', + }, + { + include: '#function-parameters', + }, + { + include: '#arrow-return-type', + }, + { + include: '#possibly-arrow-return-type', + }, + ], + }, + { + name: 'meta.arrow.js.jsx', + begin: '=>', + beginCaptures: { + '0': { + name: 'storage.type.function.arrow.js.jsx', + }, + }, + end: '((?<=\\}|\\S)(?)|((?!\\{)(?=\\S)))(?!\\/[\\/\\*])', + patterns: [ + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#decl-block', + }, + { + include: '#expression', + }, + ], + }, + ], + }, + 'indexer-declaration': { + name: 'meta.indexer.declaration.js.jsx', + begin: '(?:(?]|^await|[^\\._$[:alnum:]]await|^return|[^\\._$[:alnum:]]return|^yield|[^\\._$[:alnum:]]yield|^throw|[^\\._$[:alnum:]]throw|^in|[^\\._$[:alnum:]]in|^of|[^\\._$[:alnum:]]of|^typeof|[^\\._$[:alnum:]]typeof|&&|\\|\\||\\*)\\s*(\\{)', + beginCaptures: { + '1': { + name: 'punctuation.definition.block.js.jsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.js.jsx', + }, + }, + patterns: [ + { + include: '#object-member', + }, + ], + }, + 'object-literal': { + name: 'meta.objectliteral.js.jsx', + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.js.jsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.js.jsx', + }, + }, + patterns: [ + { + include: '#object-member', + }, + ], + }, + 'object-member': { + patterns: [ + { + include: '#comment', + }, + { + include: '#object-literal-method-declaration', + }, + { + name: 'meta.object.member.js.jsx meta.object-literal.key.js.jsx', + begin: '(?=\\[)', + end: '(?=:)|((?<=[\\]])(?=\\s*[\\(\\<]))', + patterns: [ + { + include: '#comment', + }, + { + include: '#array-literal', + }, + ], + }, + { + name: 'meta.object.member.js.jsx meta.object-literal.key.js.jsx', + begin: '(?=[\\\'\\"\\`])', + end: '(?=:)|((?<=[\\\'\\"\\`])(?=((\\s*[\\(\\<,}])|(\\s+(as|satisifies)\\s+))))', + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + ], + }, + { + name: 'meta.object.member.js.jsx meta.object-literal.key.js.jsx', + begin: '(?x)(?=(\\b(?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '0': { + name: 'meta.object-literal.key.js.jsx', + }, + '1': { + name: 'entity.name.function.js.jsx', + }, + }, + }, + { + name: 'meta.object.member.js.jsx', + match: '(?:[_$[:alpha:]][_$[:alnum:]]*)\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*:)', + captures: { + '0': { + name: 'meta.object-literal.key.js.jsx', + }, + }, + }, + { + name: 'meta.object.member.js.jsx', + begin: '\\.\\.\\.', + beginCaptures: { + '0': { + name: 'keyword.operator.spread.js.jsx', + }, + }, + end: '(?=,|\\})', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + name: 'meta.object.member.js.jsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?=,|\\}|$|\\/\\/|\\/\\*)', + captures: { + '1': { + name: 'variable.other.readwrite.js.jsx', + }, + }, + }, + { + name: 'meta.object.member.js.jsx', + match: '(?]|\\|\\||\\&\\&|\\!\\=\\=|$|^|((?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)\\(\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#type-parameters', + }, + { + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.js.jsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.js.jsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + ], + }, + { + begin: '(?<=:)\\s*(async)?\\s*(\\()(?=\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + '2': { + name: 'meta.brace.round.js.jsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.js.jsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + { + begin: '(?<=:)\\s*(async)?\\s*(?=\\<\\s*$)', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + }, + end: '(?<=\\>)', + patterns: [ + { + include: '#type-parameters', + }, + ], + }, + { + begin: '(?<=\\>)\\s*(\\()(?=\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'meta.brace.round.js.jsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.js.jsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + { + include: '#possibly-arrow-return-type', + }, + { + include: '#expression', + }, + ], + }, + { + include: '#punctuation-comma', + }, + { + include: '#decl-block', + }, + ], + }, + 'ternary-expression': { + begin: '(?!\\?\\.\\s*[^[:digit:]])(\\?)(?!\\?)', + beginCaptures: { + '1': { + name: 'keyword.operator.ternary.js.jsx', + }, + }, + end: '\\s*(:)', + endCaptures: { + '1': { + name: 'keyword.operator.ternary.js.jsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + }, + 'function-call': { + patterns: [ + { + begin: '(?=(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))\\s*(?:(\\?\\.\\s*)|(\\!))?((<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?))*(?)*(?\\s*)?\\())', + end: '(?<=\\))(?!(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))\\s*(?:(\\?\\.\\s*)|(\\!))?((<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?))*(?)*(?\\s*)?\\())', + patterns: [ + { + name: 'meta.function-call.js.jsx', + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))', + end: '(?=\\s*(?:(\\?\\.\\s*)|(\\!))?((<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?))*(?)*(?\\s*)?\\())', + patterns: [ + { + include: '#function-call-target', + }, + ], + }, + { + include: '#comment', + }, + { + include: '#function-call-optionals', + }, + { + include: '#type-arguments', + }, + { + include: '#paren-expression', + }, + ], + }, + { + begin: '(?=(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))(<\\s*[\\{\\[\\(]\\s*$))', + end: '(?<=\\>)(?!(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))(<\\s*[\\{\\[\\(]\\s*$))', + patterns: [ + { + name: 'meta.function-call.js.jsx', + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))', + end: '(?=(<\\s*[\\{\\[\\(]\\s*$))', + patterns: [ + { + include: '#function-call-target', + }, + ], + }, + { + include: '#comment', + }, + { + include: '#function-call-optionals', + }, + { + include: '#type-arguments', + }, + ], + }, + ], + }, + 'function-call-target': { + patterns: [ + { + include: '#support-function-call-identifiers', + }, + { + name: 'entity.name.function.js.jsx', + match: '(\\#?[_$[:alpha:]][_$[:alnum:]]*)', + }, + ], + }, + 'function-call-optionals': { + patterns: [ + { + name: 'meta.function-call.js.jsx punctuation.accessor.optional.js.jsx', + match: '\\?\\.', + }, + { + name: 'meta.function-call.js.jsx keyword.operator.definiteassignment.js.jsx', + match: '\\!', + }, + ], + }, + 'support-function-call-identifiers': { + patterns: [ + { + include: '#literal', + }, + { + include: '#support-objects', + }, + { + include: '#object-identifiers', + }, + { + include: '#punctuation-accessor', + }, + { + name: 'keyword.operator.expression.import.js.jsx', + match: '(?:(?]|\\|\\||\\&\\&|\\!\\=\\=|$|((?]|\\|\\||\\&\\&|\\!\\=\\=|$|(===|!==|==|!=)|(([\\&\\~\\^\\|]\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s+instanceof(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))|((?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?\\(\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#paren-expression-possibly-arrow-with-typeparameters', + }, + ], + }, + { + begin: '(?<=[(=,]|=>|^return|[^\\._$[:alnum:]]return)\\s*(async)?(?=\\s*((((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?\\()|(<)|((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)))\\s*$)', + beginCaptures: { + '1': { + name: 'storage.modifier.async.js.jsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#paren-expression-possibly-arrow-with-typeparameters', + }, + ], + }, + { + include: '#possibly-arrow-return-type', + }, + ], + }, + 'paren-expression-possibly-arrow-with-typeparameters': { + patterns: [ + { + include: '#type-parameters', + }, + { + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.js.jsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.js.jsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + ], + }, + 'expression-inside-possibly-arrow-parens': { + patterns: [ + { + include: '#expressionWithoutIdentifiers', + }, + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#decorator', + }, + { + include: '#destructuring-parameter', + }, + { + match: '(?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '1': { + name: 'storage.modifier.js.jsx', + }, + '2': { + name: 'keyword.operator.rest.js.jsx', + }, + '3': { + name: 'entity.name.function.js.jsx variable.language.this.js.jsx', + }, + '4': { + name: 'entity.name.function.js.jsx', + }, + '5': { + name: 'keyword.operator.optional.js.jsx', + }, + }, + }, + { + match: '(?x)(?:(?]|\\|\\||\\&\\&|\\!\\=\\=|$|((?>=|>>>=|\\|=', + }, + { + name: 'keyword.operator.bitwise.shift.js.jsx', + match: '<<|>>>|>>', + }, + { + name: 'keyword.operator.comparison.js.jsx', + match: '===|!==|==|!=', + }, + { + name: 'keyword.operator.relational.js.jsx', + match: '<=|>=|<>|<|>', + }, + { + match: '(?<=[_$[:alnum:]])(\\!)\\s*(?:(/=)|(?:(/)(?![/*])))', + captures: { + '1': { + name: 'keyword.operator.logical.js.jsx', + }, + '2': { + name: 'keyword.operator.assignment.compound.js.jsx', + }, + '3': { + name: 'keyword.operator.arithmetic.js.jsx', + }, + }, + }, + { + name: 'keyword.operator.logical.js.jsx', + match: '\\!|&&|\\|\\||\\?\\?', + }, + { + name: 'keyword.operator.bitwise.js.jsx', + match: '\\&|~|\\^|\\|', + }, + { + name: 'keyword.operator.assignment.js.jsx', + match: '\\=', + }, + { + name: 'keyword.operator.decrement.js.jsx', + match: '--', + }, + { + name: 'keyword.operator.increment.js.jsx', + match: '\\+\\+', + }, + { + name: 'keyword.operator.arithmetic.js.jsx', + match: '%|\\*|/|-|\\+', + }, + { + begin: '(?<=[_$[:alnum:])\\]])\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)+(?:(/=)|(?:(/)(?![/*]))))', + end: '(?:(/=)|(?:(/)(?!\\*([^\\*]|(\\*[^\\/]))*\\*\\/)))', + endCaptures: { + '1': { + name: 'keyword.operator.assignment.compound.js.jsx', + }, + '2': { + name: 'keyword.operator.arithmetic.js.jsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + ], + }, + { + match: '(?<=[_$[:alnum:])\\]])\\s*(?:(/=)|(?:(/)(?![/*])))', + captures: { + '1': { + name: 'keyword.operator.assignment.compound.js.jsx', + }, + '2': { + name: 'keyword.operator.arithmetic.js.jsx', + }, + }, + }, + ], + }, + 'typeof-operator': { + begin: '(?:&|{\\?]|(extends\\s+)|$|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#type-arguments', + }, + { + include: '#expression', + }, + ], + }, + literal: { + patterns: [ + { + include: '#numeric-literal', + }, + { + include: '#boolean-literal', + }, + { + include: '#null-literal', + }, + { + include: '#undefined-literal', + }, + { + include: '#numericConstant-literal', + }, + { + include: '#array-literal', + }, + { + include: '#this-literal', + }, + { + include: '#super-literal', + }, + ], + }, + 'array-literal': { + name: 'meta.array.literal.js.jsx', + begin: '\\s*(\\[)', + beginCaptures: { + '1': { + name: 'meta.brace.square.js.jsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'meta.brace.square.js.jsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'numeric-literal': { + patterns: [ + { + name: 'constant.numeric.hex.js.jsx', + match: '\\b(?]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\\())\n |\n (?:(EPSILON|MAX_SAFE_INTEGER|MAX_VALUE|MIN_SAFE_INTEGER|MIN_VALUE|NEGATIVE_INFINITY|POSITIVE_INFINITY)\\b(?!\\$)))', + captures: { + '1': { + name: 'punctuation.accessor.js.jsx', + }, + '2': { + name: 'punctuation.accessor.optional.js.jsx', + }, + '3': { + name: 'support.variable.property.js.jsx', + }, + '4': { + name: 'support.constant.js.jsx', + }, + }, + }, + { + match: '(?)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n))', + captures: { + '1': { + name: 'punctuation.accessor.js.jsx', + }, + '2': { + name: 'punctuation.accessor.optional.js.jsx', + }, + '3': { + name: 'entity.name.function.js.jsx', + }, + }, + }, + { + match: '(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(\\#?[[:upper:]][_$[:digit:][:upper:]]*)(?![_$[:alnum:]])', + captures: { + '1': { + name: 'punctuation.accessor.js.jsx', + }, + '2': { + name: 'punctuation.accessor.optional.js.jsx', + }, + '3': { + name: 'variable.other.constant.property.js.jsx', + }, + }, + }, + { + match: '(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'punctuation.accessor.js.jsx', + }, + '2': { + name: 'punctuation.accessor.optional.js.jsx', + }, + '3': { + name: 'variable.other.property.js.jsx', + }, + }, + }, + { + name: 'variable.other.constant.js.jsx', + match: '([[:upper:]][_$[:digit:][:upper:]]*)(?![_$[:alnum:]])', + }, + { + name: 'variable.other.readwrite.js.jsx', + match: '[_$[:alpha:]][_$[:alnum:]]*', + }, + ], + }, + 'object-identifiers': { + patterns: [ + { + name: 'support.class.js.jsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)(?=\\s*\\??\\.\\s*prototype\\b(?!\\$))', + }, + { + match: '(?x)(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(?:\n (\\#?[[:upper:]][_$[:digit:][:upper:]]*) |\n (\\#?[_$[:alpha:]][_$[:alnum:]]*)\n)(?=\\s*\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'punctuation.accessor.js.jsx', + }, + '2': { + name: 'punctuation.accessor.optional.js.jsx', + }, + '3': { + name: 'variable.other.constant.object.property.js.jsx', + }, + '4': { + name: 'variable.other.object.property.js.jsx', + }, + }, + }, + { + match: '(?x)(?:\n ([[:upper:]][_$[:digit:][:upper:]]*) |\n ([_$[:alpha:]][_$[:alnum:]]*)\n)(?=\\s*\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'variable.other.constant.object.js.jsx', + }, + '2': { + name: 'variable.other.object.js.jsx', + }, + }, + }, + ], + }, + 'type-annotation': { + patterns: [ + { + name: 'meta.type.annotation.js.jsx', + begin: '(:)(?=\\s*\\S)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.js.jsx', + }, + }, + end: '(?])|((?<=[\\}>\\]\\)]|[_$[:alpha:]])\\s*(?=\\{)))', + patterns: [ + { + include: '#type', + }, + ], + }, + { + name: 'meta.type.annotation.js.jsx', + begin: '(:)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.js.jsx', + }, + }, + end: '(?])|(?=^\\s*$)|((?<=[\\}>\\]\\)]|[_$[:alpha:]])\\s*(?=\\{)))', + patterns: [ + { + include: '#type', + }, + ], + }, + ], + }, + 'parameter-type-annotation': { + patterns: [ + { + name: 'meta.type.annotation.js.jsx', + begin: '(:)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.js.jsx', + }, + }, + end: '(?=[,)])|(?==[^>])', + patterns: [ + { + include: '#type', + }, + ], + }, + ], + }, + 'return-type': { + patterns: [ + { + name: 'meta.return.type.js.jsx', + begin: '(?<=\\))\\s*(:)(?=\\s*\\S)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.js.jsx', + }, + }, + end: '(?|\\{|(^\\s*(export|function|class|interface|let|var|const|import|enum|namespace|module|type|abstract|declare)\\s+))', + patterns: [ + { + include: '#arrow-return-type-body', + }, + ], + }, + 'possibly-arrow-return-type': { + begin: '(?<=\\)|^)\\s*(:)(?=\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*=>)', + beginCaptures: { + '1': { + name: 'meta.arrow.js.jsx meta.return.type.arrow.js.jsx keyword.operator.type.annotation.js.jsx', + }, + }, + end: '(?==>|\\{|(^\\s*(export|function|class|interface|let|var|const|import|enum|namespace|module|type|abstract|declare)\\s+))', + contentName: 'meta.arrow.js.jsx meta.return.type.arrow.js.jsx', + patterns: [ + { + include: '#arrow-return-type-body', + }, + ], + }, + 'arrow-return-type-body': { + patterns: [ + { + begin: '(?<=[:])(?=\\s*\\{)', + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + include: '#type-predicate-operator', + }, + { + include: '#type', + }, + ], + }, + 'type-parameters': { + name: 'meta.type.parameters.js.jsx', + begin: '(<)', + beginCaptures: { + '1': { + name: 'punctuation.definition.typeparameters.begin.js.jsx', + }, + }, + end: '(>)', + endCaptures: { + '1': { + name: 'punctuation.definition.typeparameters.end.js.jsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + name: 'storage.modifier.js.jsx', + match: '(?)', + }, + ], + }, + 'type-arguments': { + name: 'meta.type.parameters.js.jsx', + begin: '\\<', + beginCaptures: { + '0': { + name: 'punctuation.definition.typeparameters.begin.js.jsx', + }, + }, + end: '\\>', + endCaptures: { + '0': { + name: 'punctuation.definition.typeparameters.end.js.jsx', + }, + }, + patterns: [ + { + include: '#type-arguments-body', + }, + ], + }, + 'type-arguments-body': { + patterns: [ + { + match: '(?)\n ))\n ))\n)) |\n(:\\s*(?\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))))', + captures: { + '1': { + name: 'storage.modifier.js.jsx', + }, + '2': { + name: 'keyword.operator.rest.js.jsx', + }, + '3': { + name: 'entity.name.function.js.jsx variable.language.this.js.jsx', + }, + '4': { + name: 'entity.name.function.js.jsx', + }, + '5': { + name: 'keyword.operator.optional.js.jsx', + }, + }, + }, + { + match: '(?x)(?:(?)', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-parameters', + }, + ], + }, + { + name: 'meta.type.constructor.js.jsx', + begin: '(?)\n ))\n )\n )\n)', + end: '(?<=\\))', + patterns: [ + { + include: '#function-parameters', + }, + ], + }, + ], + }, + 'type-function-return-type': { + patterns: [ + { + name: 'meta.type.function.return.js.jsx', + begin: '(=>)(?=\\s*\\S)', + beginCaptures: { + '1': { + name: 'storage.type.function.arrow.js.jsx', + }, + }, + end: '(?)(?:\\?]|//|$)', + patterns: [ + { + include: '#type-function-return-type-core', + }, + ], + }, + { + name: 'meta.type.function.return.js.jsx', + begin: '=>', + beginCaptures: { + '0': { + name: 'storage.type.function.arrow.js.jsx', + }, + }, + end: '(?)(?]|//|^\\s*$)|((?<=\\S)(?=\\s*$)))', + patterns: [ + { + include: '#type-function-return-type-core', + }, + ], + }, + ], + }, + 'type-function-return-type-core': { + patterns: [ + { + include: '#comment', + }, + { + begin: '(?<==>)(?=\\s*\\{)', + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + include: '#type-predicate-operator', + }, + { + include: '#type', + }, + ], + }, + 'type-operators': { + patterns: [ + { + include: '#typeof-operator', + }, + { + include: '#type-infer', + }, + { + begin: '([&|])(?=\\s*\\{)', + beginCaptures: { + '0': { + name: 'keyword.operator.type.js.jsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + begin: '[&|]', + beginCaptures: { + '0': { + name: 'keyword.operator.type.js.jsx', + }, + }, + end: '(?=\\S)', + }, + { + name: 'keyword.operator.expression.keyof.js.jsx', + match: '(?)', + endCaptures: { + '1': { + name: 'meta.type.parameters.js.jsx punctuation.definition.typeparameters.end.js.jsx', + }, + }, + contentName: 'meta.type.parameters.js.jsx', + patterns: [ + { + include: '#type-arguments-body', + }, + ], + }, + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(<)', + beginCaptures: { + '1': { + name: 'entity.name.type.js.jsx', + }, + '2': { + name: 'meta.type.parameters.js.jsx punctuation.definition.typeparameters.begin.js.jsx', + }, + }, + end: '(>)', + endCaptures: { + '1': { + name: 'meta.type.parameters.js.jsx punctuation.definition.typeparameters.end.js.jsx', + }, + }, + contentName: 'meta.type.parameters.js.jsx', + patterns: [ + { + include: '#type-arguments-body', + }, + ], + }, + { + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))', + captures: { + '1': { + name: 'entity.name.type.module.js.jsx', + }, + '2': { + name: 'punctuation.accessor.js.jsx', + }, + '3': { + name: 'punctuation.accessor.optional.js.jsx', + }, + }, + }, + { + name: 'entity.name.type.js.jsx', + match: '[_$[:alpha:]][_$[:alnum:]]*', + }, + ], + }, + 'punctuation-comma': { + name: 'punctuation.separator.comma.js.jsx', + match: ',', + }, + 'punctuation-semicolon': { + name: 'punctuation.terminator.statement.js.jsx', + match: ';', + }, + 'punctuation-accessor': { + match: '(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))', + captures: { + '1': { + name: 'punctuation.accessor.js.jsx', + }, + '2': { + name: 'punctuation.accessor.optional.js.jsx', + }, + }, + }, + string: { + patterns: [ + { + include: '#qstring-single', + }, + { + include: '#qstring-double', + }, + { + include: '#template', + }, + ], + }, + 'qstring-double': { + name: 'string.quoted.double.js.jsx', + begin: '"', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.js.jsx', + }, + }, + end: '(")|((?:[^\\\\\\n])$)', + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.js.jsx', + }, + '2': { + name: 'invalid.illegal.newline.js.jsx', + }, + }, + patterns: [ + { + include: '#string-character-escape', + }, + ], + }, + 'qstring-single': { + name: 'string.quoted.single.js.jsx', + begin: "'", + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.js.jsx', + }, + }, + end: "(\\')|((?:[^\\\\\\n])$)", + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.js.jsx', + }, + '2': { + name: 'invalid.illegal.newline.js.jsx', + }, + }, + patterns: [ + { + include: '#string-character-escape', + }, + ], + }, + 'string-character-escape': { + name: 'constant.character.escape.js.jsx', + match: '\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)', + }, + template: { + patterns: [ + { + include: '#template-call', + }, + { + contentName: 'string.template.js.jsx', + begin: '([_$[:alpha:]][_$[:alnum:]]*)?(`)', + beginCaptures: { + '1': { + name: 'entity.name.function.tagged-template.js.jsx', + }, + '2': { + name: 'string.template.js.jsx punctuation.definition.string.template.begin.js.jsx', + }, + }, + end: '`', + endCaptures: { + '0': { + name: 'string.template.js.jsx punctuation.definition.string.template.end.js.jsx', + }, + }, + patterns: [ + { + include: '#template-substitution-element', + }, + { + include: '#string-character-escape', + }, + ], + }, + ], + }, + 'template-call': { + patterns: [ + { + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*\\s*\\??\\.\\s*)*|(\\??\\.\\s*)?)([_$[:alpha:]][_$[:alnum:]]*)(<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?))*(?)*(?\\s*)?`)', + end: '(?=`)', + patterns: [ + { + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*\\s*\\??\\.\\s*)*|(\\??\\.\\s*)?)([_$[:alpha:]][_$[:alnum:]]*))', + end: '(?=(<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?))*(?)*(?\\s*)?`)', + patterns: [ + { + include: '#support-function-call-identifiers', + }, + { + name: 'entity.name.function.tagged-template.js.jsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)', + }, + ], + }, + { + include: '#type-arguments', + }, + ], + }, + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)?\\s*(?=(<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?))*(?)*(?\\s*)`)', + beginCaptures: { + '1': { + name: 'entity.name.function.tagged-template.js.jsx', + }, + }, + end: '(?=`)', + patterns: [ + { + include: '#type-arguments', + }, + ], + }, + ], + }, + 'template-substitution-element': { + name: 'meta.template.expression.js.jsx', + begin: '\\$\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.template-expression.begin.js.jsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.template-expression.end.js.jsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + contentName: 'meta.embedded.line.js.jsx', + }, + 'type-string': { + patterns: [ + { + include: '#qstring-single', + }, + { + include: '#qstring-double', + }, + { + include: '#template-type', + }, + ], + }, + 'template-type': { + patterns: [ + { + include: '#template-call', + }, + { + contentName: 'string.template.js.jsx', + begin: '([_$[:alpha:]][_$[:alnum:]]*)?(`)', + beginCaptures: { + '1': { + name: 'entity.name.function.tagged-template.js.jsx', + }, + '2': { + name: 'string.template.js.jsx punctuation.definition.string.template.begin.js.jsx', + }, + }, + end: '`', + endCaptures: { + '0': { + name: 'string.template.js.jsx punctuation.definition.string.template.end.js.jsx', + }, + }, + patterns: [ + { + include: '#template-type-substitution-element', + }, + { + include: '#string-character-escape', + }, + ], + }, + ], + }, + 'template-type-substitution-element': { + name: 'meta.template.expression.js.jsx', + begin: '\\$\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.template-expression.begin.js.jsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.template-expression.end.js.jsx', + }, + }, + patterns: [ + { + include: '#type', + }, + ], + contentName: 'meta.embedded.line.js.jsx', + }, + regex: { + patterns: [ + { + name: 'string.regexp.js.jsx', + begin: '(?|&&|\\|\\||\\*\\/)\\s*(\\/)(?![\\/*])(?=(?:[^\\/\\\\\\[\\()]|\\\\.|\\[([^\\]\\\\]|\\\\.)+\\]|\\(([^\\)\\\\]|\\\\.)+\\))+\\/([dgimsuy]+|(?![\\/\\*])|(?=\\/\\*))(?!\\s*[a-zA-Z0-9_$]))', + beginCaptures: { + '1': { + name: 'punctuation.definition.string.begin.js.jsx', + }, + }, + end: '(/)([dgimsuy]*)', + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.js.jsx', + }, + '2': { + name: 'keyword.other.js.jsx', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + { + name: 'string.regexp.js.jsx', + begin: '((?', + captures: { + '0': { + name: 'keyword.other.back-reference.regexp', + }, + '1': { + name: 'variable.other.regexp', + }, + }, + }, + { + name: 'keyword.operator.quantifier.regexp', + match: '[?+*]|\\{(\\d+,\\d+|\\d+,|,\\d+|\\d+)\\}\\??', + }, + { + name: 'keyword.operator.or.regexp', + match: '\\|', + }, + { + name: 'meta.group.assertion.regexp', + begin: '(\\()((\\?=)|(\\?!)|(\\?<=)|(\\?))?', + beginCaptures: { + '0': { + name: 'punctuation.definition.group.regexp', + }, + '1': { + name: 'punctuation.definition.group.no-capture.regexp', + }, + '2': { + name: 'variable.other.regexp', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'punctuation.definition.group.regexp', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + { + name: 'constant.other.character-class.set.regexp', + begin: '(\\[)(\\^)?', + beginCaptures: { + '1': { + name: 'punctuation.definition.character-class.regexp', + }, + '2': { + name: 'keyword.operator.negation.regexp', + }, + }, + end: '(\\])', + endCaptures: { + '1': { + name: 'punctuation.definition.character-class.regexp', + }, + }, + patterns: [ + { + name: 'constant.other.character-class.range.regexp', + match: '(?:.|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))', + captures: { + '1': { + name: 'constant.character.numeric.regexp', + }, + '2': { + name: 'constant.character.control.regexp', + }, + '3': { + name: 'constant.character.escape.backslash.regexp', + }, + '4': { + name: 'constant.character.numeric.regexp', + }, + '5': { + name: 'constant.character.control.regexp', + }, + '6': { + name: 'constant.character.escape.backslash.regexp', + }, + }, + }, + { + include: '#regex-character-class', + }, + ], + }, + { + include: '#regex-character-class', + }, + ], + }, + 'regex-character-class': { + patterns: [ + { + name: 'constant.other.character-class.regexp', + match: '\\\\[wWsSdDtrnvf]|\\.', + }, + { + name: 'constant.character.numeric.regexp', + match: '\\\\([0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4})', + }, + { + name: 'constant.character.control.regexp', + match: '\\\\c[A-Z]', + }, + { + name: 'constant.character.escape.backslash.regexp', + match: '\\\\.', + }, + ], + }, + comment: { + patterns: [ + { + name: 'comment.block.documentation.js.jsx', + begin: '/\\*\\*(?!/)', + beginCaptures: { + '0': { + name: 'punctuation.definition.comment.js.jsx', + }, + }, + end: '\\*/', + endCaptures: { + '0': { + name: 'punctuation.definition.comment.js.jsx', + }, + }, + patterns: [ + { + include: '#docblock', + }, + ], + }, + { + name: 'comment.block.js.jsx', + begin: '(/\\*)(?:\\s*((@)internal)(?=\\s|(\\*/)))?', + beginCaptures: { + '1': { + name: 'punctuation.definition.comment.js.jsx', + }, + '2': { + name: 'storage.type.internaldeclaration.js.jsx', + }, + '3': { + name: 'punctuation.decorator.internaldeclaration.js.jsx', + }, + }, + end: '\\*/', + endCaptures: { + '0': { + name: 'punctuation.definition.comment.js.jsx', + }, + }, + }, + { + begin: '(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)', + beginCaptures: { + '1': { + name: 'punctuation.whitespace.comment.leading.js.jsx', + }, + '2': { + name: 'comment.line.double-slash.js.jsx', + }, + '3': { + name: 'punctuation.definition.comment.js.jsx', + }, + '4': { + name: 'storage.type.internaldeclaration.js.jsx', + }, + '5': { + name: 'punctuation.decorator.internaldeclaration.js.jsx', + }, + }, + end: '(?=$)', + contentName: 'comment.line.double-slash.js.jsx', + }, + ], + }, + 'single-line-comment-consuming-line-ending': { + begin: '(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)', + beginCaptures: { + '1': { + name: 'punctuation.whitespace.comment.leading.js.jsx', + }, + '2': { + name: 'comment.line.double-slash.js.jsx', + }, + '3': { + name: 'punctuation.definition.comment.js.jsx', + }, + '4': { + name: 'storage.type.internaldeclaration.js.jsx', + }, + '5': { + name: 'punctuation.decorator.internaldeclaration.js.jsx', + }, + }, + end: '(?=^)', + contentName: 'comment.line.double-slash.js.jsx', + }, + directives: { + name: 'comment.line.triple-slash.directive.js.jsx', + begin: '^(///)\\s*(?=<(reference|amd-dependency|amd-module)(\\s+(path|types|no-default-lib|lib|name|resolution-mode)\\s*=\\s*((\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)))+\\s*/>\\s*$)', + beginCaptures: { + '1': { + name: 'punctuation.definition.comment.js.jsx', + }, + }, + end: '(?=$)', + patterns: [ + { + name: 'meta.tag.js.jsx', + begin: '(<)(reference|amd-dependency|amd-module)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.directive.js.jsx', + }, + '2': { + name: 'entity.name.tag.directive.js.jsx', + }, + }, + end: '/>', + endCaptures: { + '0': { + name: 'punctuation.definition.tag.directive.js.jsx', + }, + }, + patterns: [ + { + name: 'entity.other.attribute-name.directive.js.jsx', + match: 'path|types|no-default-lib|lib|name|resolution-mode', + }, + { + name: 'keyword.operator.assignment.js.jsx', + match: '=', + }, + { + include: '#string', + }, + ], + }, + ], + }, + docblock: { + patterns: [ + { + match: '(?x)\n((@)(?:access|api))\n\\s+\n(private|protected|public)\n\\b', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'constant.language.access-type.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)author)\n\\s+\n(\n [^@\\s<>*/]\n (?:[^@<>*/]|\\*[^/])*\n)\n(?:\n \\s*\n (<)\n ([^>\\s]+)\n (>)\n)?', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'entity.name.type.instance.jsdoc', + }, + '4': { + name: 'punctuation.definition.bracket.angle.begin.jsdoc', + }, + '5': { + name: 'constant.other.email.link.underline.jsdoc', + }, + '6': { + name: 'punctuation.definition.bracket.angle.end.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)borrows) \\s+\n((?:[^@\\s*/]|\\*[^/])+) # \n\\s+ (as) \\s+ # as\n((?:[^@\\s*/]|\\*[^/])+) # ', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'entity.name.type.instance.jsdoc', + }, + '4': { + name: 'keyword.operator.control.jsdoc', + }, + '5': { + name: 'entity.name.type.instance.jsdoc', + }, + }, + }, + { + name: 'meta.example.jsdoc', + begin: '((@)example)\\s+', + end: '(?=@|\\*/)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + patterns: [ + { + match: '^\\s\\*\\s+', + }, + { + contentName: 'constant.other.description.jsdoc', + begin: '\\G(<)caption(>)', + beginCaptures: { + '0': { + name: 'entity.name.tag.inline.jsdoc', + }, + '1': { + name: 'punctuation.definition.bracket.angle.begin.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.angle.end.jsdoc', + }, + }, + end: '()|(?=\\*/)', + endCaptures: { + '0': { + name: 'entity.name.tag.inline.jsdoc', + }, + '1': { + name: 'punctuation.definition.bracket.angle.begin.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.angle.end.jsdoc', + }, + }, + }, + { + match: '[^\\s@*](?:[^*]|\\*[^/])*', + captures: { + '0': { + name: 'source.embedded.js.jsx', + }, + }, + }, + ], + }, + { + match: '(?x) ((@)kind) \\s+ (class|constant|event|external|file|function|member|mixin|module|namespace|typedef) \\b', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'constant.language.symbol-type.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)see)\n\\s+\n(?:\n # URL\n (\n (?=https?://)\n (?:[^\\s*]|\\*[^/])+\n )\n |\n # JSDoc namepath\n (\n (?!\n # Avoid matching bare URIs (also acceptable as links)\n https?://\n |\n # Avoid matching {@inline tags}; we match those below\n (?:\\[[^\\[\\]]*\\])? # Possible description [preceding]{@tag}\n {@(?:link|linkcode|linkplain|tutorial)\\b\n )\n # Matched namepath\n (?:[^@\\s*/]|\\*[^/])+\n )\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.link.underline.jsdoc', + }, + '4': { + name: 'entity.name.type.instance.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)template)\n\\s+\n# One or more valid identifiers\n(\n [A-Za-z_$] # First character: non-numeric word character\n [\\w$.\\[\\]]* # Rest of identifier\n (?: # Possible list of additional identifiers\n \\s* , \\s*\n [A-Za-z_$]\n [\\w$.\\[\\]]*\n )*\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + }, + }, + { + begin: '(?x)((@)template)\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + { + name: 'variable.other.jsdoc', + match: '([A-Za-z_$][\\w$.\\[\\]]*)', + }, + ], + }, + { + match: '(?x)\n(\n (@)\n (?:arg|argument|const|constant|member|namespace|param|var)\n)\n\\s+\n(\n [A-Za-z_$]\n [\\w$.\\[\\]]*\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + }, + }, + { + begin: '((@)typedef)\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + { + name: 'entity.name.type.instance.jsdoc', + match: '(?:[^@\\s*/]|\\*[^/])+', + }, + ], + }, + { + begin: '((@)(?:arg|argument|const|constant|member|namespace|param|prop|property|var))\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + { + name: 'variable.other.jsdoc', + match: '([A-Za-z_$][\\w$.\\[\\]]*)', + }, + { + name: 'variable.other.jsdoc', + match: '(?x)\n(\\[)\\s*\n[\\w$]+\n(?:\n (?:\\[\\])? # Foo[ ].bar properties within an array\n \\. # Foo.Bar namespaced parameter\n [\\w$]+\n)*\n(?:\n \\s*\n (=) # [foo=bar] Default parameter value\n \\s*\n (\n # The inner regexes are to stop the match early at */ and to not stop at escaped quotes\n (?>\n "(?:(?:\\*(?!/))|(?:\\\\(?!"))|[^*\\\\])*?" | # [foo="bar"] Double-quoted\n \'(?:(?:\\*(?!/))|(?:\\\\(?!\'))|[^*\\\\])*?\' | # [foo=\'bar\'] Single-quoted\n \\[ (?:(?:\\*(?!/))|[^*])*? \\] | # [foo=[1,2]] Array literal\n (?:(?:\\*(?!/))|\\s(?!\\s*\\])|\\[.*?(?:\\]|(?=\\*/))|[^*\\s\\[\\]])* # Everything else\n )*\n )\n)?\n\\s*(?:(\\])((?:[^*\\s]|\\*[^\\s/])+)?|(?=\\*/))', + captures: { + '1': { + name: 'punctuation.definition.optional-value.begin.bracket.square.jsdoc', + }, + '2': { + name: 'keyword.operator.assignment.jsdoc', + }, + '3': { + name: 'source.embedded.js.jsx', + }, + '4': { + name: 'punctuation.definition.optional-value.end.bracket.square.jsdoc', + }, + '5': { + name: 'invalid.illegal.syntax.jsdoc', + }, + }, + }, + ], + }, + { + begin: '(?x)\n(\n (@)\n (?:define|enum|exception|export|extends|lends|implements|modifies\n |namespace|private|protected|returns?|satisfies|suppress|this|throws|type\n |yields?)\n)\n\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + ], + }, + { + match: '(?x)\n(\n (@)\n (?:alias|augments|callback|constructs|emits|event|fires|exports?\n |extends|external|function|func|host|lends|listens|interface|memberof!?\n |method|module|mixes|mixin|name|requires|see|this|typedef|uses)\n)\n\\s+\n(\n (?:\n [^{}@\\s*] | \\*[^/]\n )+\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'entity.name.type.instance.jsdoc', + }, + }, + }, + { + contentName: 'variable.other.jsdoc', + begin: "((@)(?:default(?:value)?|license|version))\\s+(([''\"]))", + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + '4': { + name: 'punctuation.definition.string.begin.jsdoc', + }, + }, + end: '(\\3)|(?=$|\\*/)', + endCaptures: { + '0': { + name: 'variable.other.jsdoc', + }, + '1': { + name: 'punctuation.definition.string.end.jsdoc', + }, + }, + }, + { + match: '((@)(?:default(?:value)?|license|tutorial|variation|version))\\s+([^\\s*]+)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + }, + }, + { + name: 'storage.type.class.jsdoc', + match: '(?x) (@) (?:abstract|access|alias|api|arg|argument|async|attribute|augments|author|beta|borrows|bubbles |callback|chainable|class|classdesc|code|config|const|constant|constructor|constructs|copyright |default|defaultvalue|define|deprecated|desc|description|dict|emits|enum|event|example|exception |exports?|extends|extension(?:_?for)?|external|externs|file|fileoverview|final|fires|for|func |function|generator|global|hideconstructor|host|ignore|implements|implicitCast|inherit[Dd]oc |inner|instance|interface|internal|kind|lends|license|listens|main|member|memberof!?|method |mixes|mixins?|modifies|module|name|namespace|noalias|nocollapse|nocompile|nosideeffects |override|overview|package|param|polymer(?:Behavior)?|preserve|private|prop|property|protected |public|read[Oo]nly|record|require[ds]|returns?|see|since|static|struct|submodule|summary |suppress|template|this|throws|todo|tutorial|type|typedef|unrestricted|uses|var|variation |version|virtual|writeOnce|yields?) \\b', + captures: { + '1': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + }, + { + include: '#inline-tags', + }, + { + match: '((@)(?:[_$[:alpha:]][_$[:alnum:]]*))(?=\\s+)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + }, + ], + }, + brackets: { + patterns: [ + { + begin: '{', + end: '}|(?=\\*/)', + patterns: [ + { + include: '#brackets', + }, + ], + }, + { + begin: '\\[', + end: '\\]|(?=\\*/)', + patterns: [ + { + include: '#brackets', + }, + ], + }, + ], + }, + 'inline-tags': { + patterns: [ + { + name: 'constant.other.description.jsdoc', + match: '(\\[)[^\\]]+(\\])(?={@(?:link|linkcode|linkplain|tutorial))', + captures: { + '1': { + name: 'punctuation.definition.bracket.square.begin.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.square.end.jsdoc', + }, + }, + }, + { + name: 'entity.name.type.instance.jsdoc', + begin: '({)((@)(?:link(?:code|plain)?|tutorial))\\s*', + beginCaptures: { + '1': { + name: 'punctuation.definition.bracket.curly.begin.jsdoc', + }, + '2': { + name: 'storage.type.class.jsdoc', + }, + '3': { + name: 'punctuation.definition.inline.tag.jsdoc', + }, + }, + end: '}|(?=\\*/)', + endCaptures: { + '0': { + name: 'punctuation.definition.bracket.curly.end.jsdoc', + }, + }, + patterns: [ + { + match: '\\G((?=https?://)(?:[^|}\\s*]|\\*[/])+)(\\|)?', + captures: { + '1': { + name: 'variable.other.link.underline.jsdoc', + }, + '2': { + name: 'punctuation.separator.pipe.jsdoc', + }, + }, + }, + { + match: '\\G((?:[^{}@\\s|*]|\\*[^/])+)(\\|)?', + captures: { + '1': { + name: 'variable.other.description.jsdoc', + }, + '2': { + name: 'punctuation.separator.pipe.jsdoc', + }, + }, + }, + ], + }, + ], + }, + jsdoctype: { + patterns: [ + { + contentName: 'entity.name.type.instance.jsdoc', + begin: '\\G({)', + beginCaptures: { + '0': { + name: 'entity.name.type.instance.jsdoc', + }, + '1': { + name: 'punctuation.definition.bracket.curly.begin.jsdoc', + }, + }, + end: '((}))\\s*|(?=\\*/)', + endCaptures: { + '1': { + name: 'entity.name.type.instance.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.curly.end.jsdoc', + }, + }, + patterns: [ + { + include: '#brackets', + }, + ], + }, + ], + }, + jsx: { + patterns: [ + { + include: '#jsx-tag-without-attributes-in-expression', + }, + { + include: '#jsx-tag-in-expression', + }, + ], + }, + 'jsx-tag-without-attributes-in-expression': { + begin: '(?:*]|&&|\\|\\||\\?|\\*\\/|^await|[^\\._$[:alnum:]]await|^return|[^\\._$[:alnum:]]return|^default|[^\\._$[:alnum:]]default|^yield|[^\\._$[:alnum:]]yield|^)\\s*(?=(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?))', + end: '(?!(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?))', + patterns: [ + { + include: '#jsx-tag-without-attributes', + }, + ], + }, + 'jsx-tag-without-attributes': { + name: 'meta.tag.without-attributes.js.jsx', + begin: '(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?)', + end: '()', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.begin.js.jsx', + }, + '2': { + name: 'entity.name.tag.namespace.js.jsx', + }, + '3': { + name: 'punctuation.separator.namespace.js.jsx', + }, + '4': { + name: 'entity.name.tag.js.jsx', + }, + '5': { + name: 'support.class.component.js.jsx', + }, + '6': { + name: 'punctuation.definition.tag.end.js.jsx', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.definition.tag.begin.js.jsx', + }, + '2': { + name: 'entity.name.tag.namespace.js.jsx', + }, + '3': { + name: 'punctuation.separator.namespace.js.jsx', + }, + '4': { + name: 'entity.name.tag.js.jsx', + }, + '5': { + name: 'support.class.component.js.jsx', + }, + '6': { + name: 'punctuation.definition.tag.end.js.jsx', + }, + }, + contentName: 'meta.jsx.children.js.jsx', + patterns: [ + { + include: '#jsx-children', + }, + ], + }, + 'jsx-tag-in-expression': { + begin: '(?x)\n (?:*]|&&|\\|\\||\\?|\\*\\/|^await|[^\\._$[:alnum:]]await|^return|[^\\._$[:alnum:]]return|^default|[^\\._$[:alnum:]]default|^yield|[^\\._$[:alnum:]]yield|^)\\s*\n (?!<\\s*[_$[:alpha:]][_$[:alnum:]]*((\\s+extends\\s+[^=>])|,)) # look ahead is not type parameter of arrow\n (?=(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?))', + end: '(?!(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?))', + patterns: [ + { + include: '#jsx-tag', + }, + ], + }, + 'jsx-tag': { + name: 'meta.tag.js.jsx', + begin: '(?=(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?))', + end: '(/>)|(?:())', + endCaptures: { + '1': { + name: 'punctuation.definition.tag.end.js.jsx', + }, + '2': { + name: 'punctuation.definition.tag.begin.js.jsx', + }, + '3': { + name: 'entity.name.tag.namespace.js.jsx', + }, + '4': { + name: 'punctuation.separator.namespace.js.jsx', + }, + '5': { + name: 'entity.name.tag.js.jsx', + }, + '6': { + name: 'support.class.component.js.jsx', + }, + '7': { + name: 'punctuation.definition.tag.end.js.jsx', + }, + }, + patterns: [ + { + begin: '(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.begin.js.jsx', + }, + '2': { + name: 'entity.name.tag.namespace.js.jsx', + }, + '3': { + name: 'punctuation.separator.namespace.js.jsx', + }, + '4': { + name: 'entity.name.tag.js.jsx', + }, + '5': { + name: 'support.class.component.js.jsx', + }, + }, + end: '(?=[/]?>)', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-arguments', + }, + { + include: '#jsx-tag-attributes', + }, + ], + }, + { + begin: '(>)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.end.js.jsx', + }, + }, + end: '(?=)', + patterns: [ + { + include: '#comment', + }, + { + include: '#jsx-tag-attribute-name', + }, + { + include: '#jsx-tag-attribute-assignment', + }, + { + include: '#jsx-string-double-quoted', + }, + { + include: '#jsx-string-single-quoted', + }, + { + include: '#jsx-evaluated-code', + }, + { + include: '#jsx-tag-attributes-illegal', + }, + ], + }, + 'jsx-tag-attribute-name': { + match: '(?x)\n \\s*\n (?:([_$[:alpha:]][-_$[:alnum:].]*)(:))?\n ([_$[:alpha:]][-_$[:alnum:]]*)\n (?=\\s|=|/?>|/\\*|//)', + captures: { + '1': { + name: 'entity.other.attribute-name.namespace.js.jsx', + }, + '2': { + name: 'punctuation.separator.namespace.js.jsx', + }, + '3': { + name: 'entity.other.attribute-name.js.jsx', + }, + }, + }, + 'jsx-tag-attribute-assignment': { + name: 'keyword.operator.assignment.js.jsx', + match: '=(?=\\s*(?:\'|"|{|/\\*|//|\\n))', + }, + 'jsx-string-double-quoted': { + name: 'string.quoted.double.js.jsx', + begin: '"', + end: '"', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.js.jsx', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.string.end.js.jsx', + }, + }, + patterns: [ + { + include: '#jsx-entities', + }, + ], + }, + 'jsx-string-single-quoted': { + name: 'string.quoted.single.js.jsx', + begin: "'", + end: "'", + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.js.jsx', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.string.end.js.jsx', + }, + }, + patterns: [ + { + include: '#jsx-entities', + }, + ], + }, + 'jsx-tag-attributes-illegal': { + name: 'invalid.illegal.attribute.js.jsx', + match: '\\S+', + }, + }, +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/languages/markdown-latex-combined.tmLanguage.ts b/completions-sample-code/vscode-node/extension/src/panelShared/languages/markdown-latex-combined.tmLanguage.ts new file mode 100644 index 0000000..dfa4c24 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/languages/markdown-latex-combined.tmLanguage.ts @@ -0,0 +1,3011 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/* eslint-disable local/no-unexternalized-strings */ +import { LanguageInput } from 'shiki/core'; + +// This file has been converted from https://github.com/jlelong/vscode-latex-basics/blob/master/syntaxes/markdown-latex-combined.tmLanguage.json +// If you want to provide a fix or improvement, please create a pull request against the original repository. +// Once accepted there, we are happy to receive an update request. +// version: https://github.com/jlelong/vscode-latex-basics/commit/002278bd5484c278587a2aa3cafc1616538a20bc +export const markdownLatexCombined: LanguageInput = { + name: 'markdown_latex_combined', + scopeName: 'text.tex.markdown_latex_combined', + patterns: [ + { + include: 'text.tex.latex', + }, + { + include: '#frontMatter', + }, + { + include: '#block', + }, + ], + repository: { + $self: {}, + $base: {}, + block: { + patterns: [ + { + include: '#separator', + }, + { + include: '#heading', + }, + { + include: '#blockquote', + }, + { + include: '#lists', + }, + { + include: '#fenced_code_block', + }, + { + include: '#raw_block', + }, + { + include: '#link-def', + }, + { + include: '#html', + }, + { + include: '#paragraph', + }, + ], + }, + blockquote: { + begin: '(^|\\G)[ ]{0,3}(>) ?', + captures: { + '2': { + name: 'punctuation.definition.quote.begin.markdown', + }, + }, + name: 'markup.quote.markdown', + patterns: [ + { + include: '#block', + }, + ], + while: '(^|\\G)\\s*(>) ?', + }, + fenced_code_block_css: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(css|css.erb)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.css', + patterns: [ + { + include: 'source.css', + }, + ], + }, + ], + }, + fenced_code_block_basic: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(html|htm|shtml|xhtml|inc|tmpl|tpl)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.html', + patterns: [ + { + include: 'text.html.basic', + }, + ], + }, + ], + }, + fenced_code_block_ini: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(ini|conf)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.ini', + patterns: [ + { + include: 'source.ini', + }, + ], + }, + ], + }, + fenced_code_block_java: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(java|bsh)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.java', + patterns: [ + { + include: 'source.java', + }, + ], + }, + ], + }, + fenced_code_block_lua: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(lua)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.lua', + patterns: [ + { + include: 'source.lua', + }, + ], + }, + ], + }, + fenced_code_block_makefile: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(Makefile|makefile|GNUmakefile|OCamlMakefile)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.makefile', + patterns: [ + { + include: 'source.makefile', + }, + ], + }, + ], + }, + fenced_code_block_perl: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(perl|pl|pm|pod|t|PL|psgi|vcl)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.perl', + patterns: [ + { + include: 'source.perl', + }, + ], + }, + ], + }, + fenced_code_block_r: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(R|r|s|S|Rprofile|\\{\\.r.+?\\})((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.r', + patterns: [ + { + include: 'source.r', + }, + ], + }, + ], + }, + fenced_code_block_ruby: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(ruby|rb|rbx|rjs|Rakefile|rake|cgi|fcgi|gemspec|irbrc|Capfile|ru|prawn|Cheffile|Gemfile|Guardfile|Hobofile|Vagrantfile|Appraisals|Rantfile|Berksfile|Berksfile.lock|Thorfile|Puppetfile)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.ruby', + patterns: [ + { + include: 'source.ruby', + }, + ], + }, + ], + }, + fenced_code_block_php: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(php|php3|php4|php5|phpt|phtml|aw|ctp)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.php', + patterns: [ + { + include: 'text.html.basic', + }, + { + include: 'source.php', + }, + ], + }, + ], + }, + fenced_code_block_sql: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(sql|ddl|dml)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.sql', + patterns: [ + { + include: 'source.sql', + }, + ], + }, + ], + }, + fenced_code_block_vs_net: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(vb)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.vs_net', + patterns: [ + { + include: 'source.asp.vb.net', + }, + ], + }, + ], + }, + fenced_code_block_xml: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(xml|xsd|tld|jsp|pt|cpt|dtml|rss|opml)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.xml', + patterns: [ + { + include: 'text.xml', + }, + ], + }, + ], + }, + fenced_code_block_xsl: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(xsl|xslt)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.xsl', + patterns: [ + { + include: 'text.xml.xsl', + }, + ], + }, + ], + }, + fenced_code_block_yaml: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(yaml|yml)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.yaml', + patterns: [ + { + include: 'source.yaml', + }, + ], + }, + ], + }, + fenced_code_block_dosbatch: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(bat|batch)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.dosbatch', + patterns: [ + { + include: 'source.batchfile', + }, + ], + }, + ], + }, + fenced_code_block_clojure: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(clj|cljs|clojure)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.clojure', + patterns: [ + { + include: 'source.clojure', + }, + ], + }, + ], + }, + fenced_code_block_coffee: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(coffee|Cakefile|coffee.erb)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.coffee', + patterns: [ + { + include: 'source.coffee', + }, + ], + }, + ], + }, + fenced_code_block_c: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(c|h)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.c', + patterns: [ + { + include: 'source.c', + }, + ], + }, + ], + }, + fenced_code_block_cpp: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(cpp|c\\+\\+|cxx)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.cpp source.cpp', + patterns: [ + { + include: 'source.cpp', + }, + ], + }, + ], + }, + fenced_code_block_diff: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(patch|diff|rej)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.diff', + patterns: [ + { + include: 'source.diff', + }, + ], + }, + ], + }, + fenced_code_block_dockerfile: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(dockerfile|Dockerfile)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.dockerfile', + patterns: [ + { + include: 'source.dockerfile', + }, + ], + }, + ], + }, + fenced_code_block_git_commit: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(COMMIT_EDITMSG|MERGE_MSG)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.git_commit', + patterns: [ + { + include: 'text.git-commit', + }, + ], + }, + ], + }, + fenced_code_block_git_rebase: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(git-rebase-todo)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.git_rebase', + patterns: [ + { + include: 'text.git-rebase', + }, + ], + }, + ], + }, + fenced_code_block_go: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(go|golang)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.go', + patterns: [ + { + include: 'source.go', + }, + ], + }, + ], + }, + fenced_code_block_groovy: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(groovy|gvy)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.groovy', + patterns: [ + { + include: 'source.groovy', + }, + ], + }, + ], + }, + fenced_code_block_pug: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(jade|pug)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.pug', + patterns: [ + { + include: 'text.pug', + }, + ], + }, + ], + }, + fenced_code_block_js: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(js|jsx|javascript|es6|mjs|cjs|dataviewjs|\\{\\.js.+?\\})((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.javascript', + patterns: [ + { + include: 'source.js', + }, + ], + }, + ], + }, + fenced_code_block_js_regexp: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(regexp)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.js_regexp', + patterns: [ + { + include: 'source.js.regexp', + }, + ], + }, + ], + }, + fenced_code_block_json: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(json|json5|sublime-settings|sublime-menu|sublime-keymap|sublime-mousemap|sublime-theme|sublime-build|sublime-project|sublime-completions)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.json', + patterns: [ + { + include: 'source.json', + }, + ], + }, + ], + }, + fenced_code_block_jsonc: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(jsonc)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.jsonc', + patterns: [ + { + include: 'source.json.comments', + }, + ], + }, + ], + }, + fenced_code_block_less: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(less)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.less', + patterns: [ + { + include: 'source.css.less', + }, + ], + }, + ], + }, + fenced_code_block_objc: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(objectivec|objective-c|mm|objc|obj-c|m|h)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.objc', + patterns: [ + { + include: 'source.objc', + }, + ], + }, + ], + }, + fenced_code_block_swift: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(swift)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.swift', + patterns: [ + { + include: 'source.swift', + }, + ], + }, + ], + }, + fenced_code_block_scss: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(scss)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.scss', + patterns: [ + { + include: 'source.css.scss', + }, + ], + }, + ], + }, + fenced_code_block_perl6: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(perl6|p6|pl6|pm6|nqp)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.perl6', + patterns: [ + { + include: 'source.perl.6', + }, + ], + }, + ], + }, + fenced_code_block_powershell: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(powershell|ps1|psm1|psd1)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.powershell', + patterns: [ + { + include: 'source.powershell', + }, + ], + }, + ], + }, + fenced_code_block_python: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(python|py|py3|rpy|pyw|cpy|SConstruct|Sconstruct|sconstruct|SConscript|gyp|gypi|\\{\\.python.+?\\})((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.python', + patterns: [ + { + include: 'source.python', + }, + ], + }, + ], + }, + fenced_code_block_julia: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(julia|\\{\\.julia.+?\\})((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.julia', + patterns: [ + { + include: 'source.julia', + }, + ], + }, + ], + }, + fenced_code_block_regexp_python: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(re)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.regexp_python', + patterns: [ + { + include: 'source.regexp.python', + }, + ], + }, + ], + }, + fenced_code_block_rust: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(rust|rs|\\{\\.rust.+?\\})((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.rust', + patterns: [ + { + include: 'source.rust', + }, + ], + }, + ], + }, + fenced_code_block_scala: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(scala|sbt)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.scala', + patterns: [ + { + include: 'source.scala', + }, + ], + }, + ], + }, + fenced_code_block_shell: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(shell|sh|bash|zsh|bashrc|bash_profile|bash_login|profile|bash_logout|.textmate_init|\\{\\.bash.+?\\})((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.shellscript', + patterns: [ + { + include: 'source.shell', + }, + ], + }, + ], + }, + fenced_code_block_ts: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(typescript|ts)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.typescript', + patterns: [ + { + include: 'source.ts', + }, + ], + }, + ], + }, + fenced_code_block_tsx: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(tsx)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.typescriptreact', + patterns: [ + { + include: 'source.tsx', + }, + ], + }, + ], + }, + fenced_code_block_csharp: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(cs|csharp|c#)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.csharp', + patterns: [ + { + include: 'source.cs', + }, + ], + }, + ], + }, + fenced_code_block_fsharp: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(fs|fsharp|f#)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.fsharp', + patterns: [ + { + include: 'source.fsharp', + }, + ], + }, + ], + }, + fenced_code_block_dart: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(dart)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.dart', + patterns: [ + { + include: 'source.dart', + }, + ], + }, + ], + }, + fenced_code_block_handlebars: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(handlebars|hbs)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.handlebars', + patterns: [ + { + include: 'text.html.handlebars', + }, + ], + }, + ], + }, + fenced_code_block_markdown: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(markdown|md)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.markdown', + patterns: [ + { + include: 'text.html.markdown', + }, + ], + }, + ], + }, + fenced_code_block_log: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(log)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.log', + patterns: [ + { + include: 'text.log', + }, + ], + }, + ], + }, + fenced_code_block_erlang: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(erlang)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.erlang', + patterns: [ + { + include: 'source.erlang', + }, + ], + }, + ], + }, + fenced_code_block_elixir: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(elixir)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.elixir', + patterns: [ + { + include: 'source.elixir', + }, + ], + }, + ], + }, + fenced_code_block_latex: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(latex|tex)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.latex', + patterns: [ + { + include: 'text.tex.latex', + }, + ], + }, + ], + }, + fenced_code_block_bibtex: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(bibtex)((\\s+|:|,|\\{|\\?)[^`~]*)?$)', + name: 'markup.fenced_code.block.markdown', + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language.markdown', + }, + '5': { + name: 'fenced_code.block.language.attributes.markdown', + }, + }, + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + patterns: [ + { + begin: '(^|\\G)(\\s*)(.*)', + while: '(^|\\G)(?!\\s*([`~]{3,})\\s*$)', + contentName: 'meta.embedded.block.bibtex', + patterns: [ + { + include: 'text.bibtex', + }, + ], + }, + ], + }, + fenced_code_block: { + patterns: [ + { + include: '#fenced_code_block_css', + }, + { + include: '#fenced_code_block_basic', + }, + { + include: '#fenced_code_block_ini', + }, + { + include: '#fenced_code_block_java', + }, + { + include: '#fenced_code_block_lua', + }, + { + include: '#fenced_code_block_makefile', + }, + { + include: '#fenced_code_block_perl', + }, + { + include: '#fenced_code_block_r', + }, + { + include: '#fenced_code_block_ruby', + }, + { + include: '#fenced_code_block_php', + }, + { + include: '#fenced_code_block_sql', + }, + { + include: '#fenced_code_block_vs_net', + }, + { + include: '#fenced_code_block_xml', + }, + { + include: '#fenced_code_block_xsl', + }, + { + include: '#fenced_code_block_yaml', + }, + { + include: '#fenced_code_block_dosbatch', + }, + { + include: '#fenced_code_block_clojure', + }, + { + include: '#fenced_code_block_coffee', + }, + { + include: '#fenced_code_block_c', + }, + { + include: '#fenced_code_block_cpp', + }, + { + include: '#fenced_code_block_diff', + }, + { + include: '#fenced_code_block_dockerfile', + }, + { + include: '#fenced_code_block_git_commit', + }, + { + include: '#fenced_code_block_git_rebase', + }, + { + include: '#fenced_code_block_go', + }, + { + include: '#fenced_code_block_groovy', + }, + { + include: '#fenced_code_block_pug', + }, + { + include: '#fenced_code_block_js', + }, + { + include: '#fenced_code_block_js_regexp', + }, + { + include: '#fenced_code_block_json', + }, + { + include: '#fenced_code_block_jsonc', + }, + { + include: '#fenced_code_block_less', + }, + { + include: '#fenced_code_block_objc', + }, + { + include: '#fenced_code_block_swift', + }, + { + include: '#fenced_code_block_scss', + }, + { + include: '#fenced_code_block_perl6', + }, + { + include: '#fenced_code_block_powershell', + }, + { + include: '#fenced_code_block_python', + }, + { + include: '#fenced_code_block_julia', + }, + { + include: '#fenced_code_block_regexp_python', + }, + { + include: '#fenced_code_block_rust', + }, + { + include: '#fenced_code_block_scala', + }, + { + include: '#fenced_code_block_shell', + }, + { + include: '#fenced_code_block_ts', + }, + { + include: '#fenced_code_block_tsx', + }, + { + include: '#fenced_code_block_csharp', + }, + { + include: '#fenced_code_block_fsharp', + }, + { + include: '#fenced_code_block_dart', + }, + { + include: '#fenced_code_block_handlebars', + }, + { + include: '#fenced_code_block_markdown', + }, + { + include: '#fenced_code_block_log', + }, + { + include: '#fenced_code_block_erlang', + }, + { + include: '#fenced_code_block_elixir', + }, + { + include: '#fenced_code_block_latex', + }, + { + include: '#fenced_code_block_bibtex', + }, + { + include: '#fenced_code_block_unknown', + }, + ], + }, + fenced_code_block_unknown: { + begin: '(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?=([^`~]*)?$)', + beginCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + '4': { + name: 'fenced_code.block.language', + }, + }, + end: '(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$', + endCaptures: { + '3': { + name: 'punctuation.definition.markdown', + }, + }, + name: 'markup.fenced_code.block.markdown', + }, + heading: { + match: '(?:^|\\G)[ ]{0,3}(#{1,6}\\s+(.*?)(\\s+#{1,6})?\\s*)$', + captures: { + '1': { + patterns: [ + { + match: '(#{6})\\s+(.*?)(?:\\s+(#+))?\\s*$', + name: 'heading.6.markdown', + captures: { + '1': { + name: 'punctuation.definition.heading.markdown', + }, + '2': { + name: 'entity.name.section.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + ], + }, + '3': { + name: 'punctuation.definition.heading.markdown', + }, + }, + }, + { + match: '(#{5})\\s+(.*?)(?:\\s+(#+))?\\s*$', + name: 'heading.5.markdown', + captures: { + '1': { + name: 'punctuation.definition.heading.markdown', + }, + '2': { + name: 'entity.name.section.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + ], + }, + '3': { + name: 'punctuation.definition.heading.markdown', + }, + }, + }, + { + match: '(#{4})\\s+(.*?)(?:\\s+(#+))?\\s*$', + name: 'heading.4.markdown', + captures: { + '1': { + name: 'punctuation.definition.heading.markdown', + }, + '2': { + name: 'entity.name.section.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + ], + }, + '3': { + name: 'punctuation.definition.heading.markdown', + }, + }, + }, + { + match: '(#{3})\\s+(.*?)(?:\\s+(#+))?\\s*$', + name: 'heading.3.markdown', + captures: { + '1': { + name: 'punctuation.definition.heading.markdown', + }, + '2': { + name: 'entity.name.section.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + ], + }, + '3': { + name: 'punctuation.definition.heading.markdown', + }, + }, + }, + { + match: '(#{2})\\s+(.*?)(?:\\s+(#+))?\\s*$', + name: 'heading.2.markdown', + captures: { + '1': { + name: 'punctuation.definition.heading.markdown', + }, + '2': { + name: 'entity.name.section.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + ], + }, + '3': { + name: 'punctuation.definition.heading.markdown', + }, + }, + }, + { + match: '(#{1})\\s+(.*?)(?:\\s+(#+))?\\s*$', + name: 'heading.1.markdown', + captures: { + '1': { + name: 'punctuation.definition.heading.markdown', + }, + '2': { + name: 'entity.name.section.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + ], + }, + '3': { + name: 'punctuation.definition.heading.markdown', + }, + }, + }, + ], + }, + }, + name: 'markup.heading.markdown', + patterns: [ + { + include: '#inline', + }, + ], + }, + 'heading-setext': { + patterns: [ + { + match: '^(={3,})(?=[ \\t]*$\\n?)', + name: 'markup.heading.setext.1.markdown', + }, + { + match: '^(-{3,})(?=[ \\t]*$\\n?)', + name: 'markup.heading.setext.2.markdown', + }, + ], + }, + html: { + patterns: [ + { + begin: '(^|\\G)\\s*()', + name: 'comment.block.html', + }, + { + begin: '(?i)(^|\\G)\\s*(?=<(script|style|pre)(\\s|$|>)(?!.*?))', + end: '(?i)(.*)(())', + endCaptures: { + '1': { + patterns: [ + { + include: 'text.html.derivative', + }, + ], + }, + '2': { + name: 'meta.tag.structure.$4.end.html', + }, + '3': { + name: 'punctuation.definition.tag.begin.html', + }, + '4': { + name: 'entity.name.tag.html', + }, + '5': { + name: 'punctuation.definition.tag.end.html', + }, + }, + patterns: [ + { + begin: '(\\s*|$)', + patterns: [ + { + include: 'text.html.derivative', + }, + ], + while: '(?i)^(?!.*)', + }, + ], + }, + { + begin: '(?i)(^|\\G)\\s*(?=))', + patterns: [ + { + include: 'text.html.derivative', + }, + ], + while: '^(?!\\s*$)', + }, + { + begin: '(^|\\G)\\s*(?=(<[a-zA-Z0-9\\-](/?>|\\s.*?>)|)\\s*$)', + patterns: [ + { + include: 'text.html.derivative', + }, + ], + while: '^(?!\\s*$)', + }, + ], + }, + 'link-def': { + captures: { + '1': { + name: 'punctuation.definition.constant.markdown', + }, + '2': { + name: 'constant.other.reference.link.markdown', + }, + '3': { + name: 'punctuation.definition.constant.markdown', + }, + '4': { + name: 'punctuation.separator.key-value.markdown', + }, + '5': { + name: 'punctuation.definition.link.markdown', + }, + '6': { + name: 'markup.underline.link.markdown', + }, + '7': { + name: 'punctuation.definition.link.markdown', + }, + '8': { + name: 'markup.underline.link.markdown', + }, + '9': { + name: 'string.other.link.description.title.markdown', + }, + '10': { + name: 'punctuation.definition.string.begin.markdown', + }, + '11': { + name: 'punctuation.definition.string.end.markdown', + }, + '12': { + name: 'string.other.link.description.title.markdown', + }, + '13': { + name: 'punctuation.definition.string.begin.markdown', + }, + '14': { + name: 'punctuation.definition.string.end.markdown', + }, + '15': { + name: 'string.other.link.description.title.markdown', + }, + '16': { + name: 'punctuation.definition.string.begin.markdown', + }, + '17': { + name: 'punctuation.definition.string.end.markdown', + }, + }, + match: '(?x)\n \\s* # Leading whitespace\n (\\[)([^]]+?)(\\])(:) # Reference name\n [ \\t]* # Optional whitespace\n (?:(<)([^\\>]+?)(>)|(\\S+?)) # The url\n [ \\t]* # Optional whitespace\n (?:\n ((\\().+?(\\))) # Match title in parensโ€ฆ\n | ((").+?(")) # or in double quotesโ€ฆ\n | ((\').+?(\')) # or in single quotes.\n )? # Title is optional\n \\s* # Optional whitespace\n $\n', + name: 'meta.link.reference.def.markdown', + }, + list_paragraph: { + begin: '(^|\\G)(?=\\S)(?![*+->]\\s|[0-9]+\\.\\s)', + name: 'meta.paragraph.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + { + include: '#heading-setext', + }, + ], + while: '(^|\\G)(?!\\s*$|#|[ ]{0,3}([-*_>][ ]{2,}){3,}[ \\t]*$\\n?|[ ]{0,3}[*+->]|[ ]{0,3}[0-9]+\\.)', + }, + lists: { + patterns: [ + { + begin: '(^|\\G)([ ]{0,3})([*+-])([ \\t])', + beginCaptures: { + '3': { + name: 'punctuation.definition.list.begin.markdown', + }, + }, + // comment: 'Currently does not support un-indented second lines.', + name: 'markup.list.unnumbered.markdown', + patterns: [ + { + include: '#block', + }, + { + include: '#list_paragraph', + }, + ], + while: '((^|\\G)([ ]{2,4}|\\t))|(^[ \\t]*$)', + }, + { + begin: '(^|\\G)([ ]{0,3})([0-9]+\\.)([ \\t])', + beginCaptures: { + '3': { + name: 'punctuation.definition.list.begin.markdown', + }, + }, + name: 'markup.list.numbered.markdown', + patterns: [ + { + include: '#block', + }, + { + include: '#list_paragraph', + }, + ], + while: '((^|\\G)([ ]{2,4}|\\t))|(^[ \\t]*$)', + }, + ], + }, + paragraph: { + begin: '(^|\\G)[ ]{0,3}(?=\\S)', + name: 'meta.paragraph.markdown', + patterns: [ + { + include: '#inline', + }, + { + include: 'text.html.derivative', + }, + { + include: '#heading-setext', + }, + ], + while: '(^|\\G)((?=\\s*[-=]{3,}\\s*$)|[ ]{4,}(?=\\S))', + }, + raw_block: { + begin: '(^|\\G)([ ]{4}|\\t)', + name: 'markup.raw.block.markdown', + while: '(^|\\G)([ ]{4}|\\t)', + }, + separator: { + match: '(^|\\G)[ ]{0,3}([\\*\\-\\_])([ ]{0,2}\\2){2,}[ \\t]*$\\n?', + name: 'meta.separator.markdown', + }, + frontMatter: { + begin: '\\A-{3}\\s*$', + contentName: 'meta.embedded.block.frontmatter', + patterns: [ + { + include: 'source.yaml', + }, + ], + end: '(^|\\G)-{3}|\\.{3}\\s*$', + }, + inline: { + patterns: [ + { + include: 'text.tex.latex', + }, + { + include: '#ampersand', + }, + { + include: '#bracket', + }, + { + include: '#bold', + }, + { + include: '#italic', + }, + { + include: '#raw', + }, + { + include: '#strikethrough', + }, + { + include: '#escape', + }, + { + include: '#image-inline', + }, + { + include: '#image-ref', + }, + { + include: '#link-email', + }, + { + include: '#link-inet', + }, + { + include: '#link-inline', + }, + { + include: '#link-ref', + }, + { + include: '#link-ref-literal', + }, + { + include: '#link-ref-shortcut', + }, + ], + }, + ampersand: { + // comment: 'Markdown will convert this for us. We match it so that the HTML grammar will not mark it up as invalid.', + match: '&(?!([a-zA-Z0-9]+|#[0-9]+|#x[0-9a-fA-F]+);)', + name: 'meta.other.valid-ampersand.markdown', + }, + bold: { + begin: '(?x) (?(\\*\\*(?=\\w)|(?]*+> # HTML tags\n | (?`+)([^`]|(?!(?(?!`))`)*+\\k\n # Raw\n | \\\\[\\\\`*_{}\\[\\]()#.!+\\->]?+ # Escapes\n | \\[\n (\n (? # Named group\n [^\\[\\]\\\\] # Match most chars\n | \\\\. # Escaped chars\n | \\[ \\g*+ \\] # Nested brackets\n )*+\n \\]\n (\n ( # Reference Link\n [ ]? # Optional space\n \\[[^\\]]*+\\] # Ref name\n )\n | ( # Inline Link\n \\( # Opening paren\n [ \\t]*+ # Optional whitespace\n ? # URL\n [ \\t]*+ # Optional whitespace\n ( # Optional Title\n (?[\'"])\n (.*?)\n \\k<title>\n )?\n \\)\n )\n )\n )\n | (?!(?<=\\S)\\k<open>). # Everything besides\n # style closer\n )++\n (?<=\\S)(?=__\\b|\\*\\*)\\k<open> # Close\n)\n', + captures: { + '1': { + name: 'punctuation.definition.bold.markdown', + }, + }, + end: '(?<=\\S)(\\1)', + name: 'markup.bold.markdown', + patterns: [ + { + applyEndPatternLast: true, + begin: '(?=<[^>]*?>)', + end: '(?<=>)', + patterns: [ + { + include: 'text.html.derivative', + }, + ], + }, + { + include: '#escape', + }, + { + include: '#ampersand', + }, + { + include: '#bracket', + }, + { + include: '#raw', + }, + { + include: '#bold', + }, + { + include: '#italic', + }, + { + include: '#image-inline', + }, + { + include: '#link-inline', + }, + { + include: '#link-inet', + }, + { + include: '#link-email', + }, + { + include: '#image-ref', + }, + { + include: '#link-ref-literal', + }, + { + include: '#link-ref', + }, + { + include: '#link-ref-shortcut', + }, + { + include: '#strikethrough', + }, + ], + }, + bracket: { + // comment: 'Markdown will convert this for us. We match it so that the HTML grammar will not mark it up as invalid.', + match: '<(?![a-zA-Z/?\\$!])', + name: 'meta.other.valid-bracket.markdown', + }, + escape: { + match: '\\\\[-`*_#+.!(){}\\[\\]\\\\>]', + name: 'constant.character.escape.markdown', + }, + 'image-inline': { + captures: { + '1': { + name: 'punctuation.definition.link.description.begin.markdown', + }, + '2': { + name: 'string.other.link.description.markdown', + }, + '4': { + name: 'punctuation.definition.link.description.end.markdown', + }, + '5': { + name: 'punctuation.definition.metadata.markdown', + }, + '6': { + name: 'punctuation.definition.link.markdown', + }, + '7': { + name: 'markup.underline.link.image.markdown', + }, + '8': { + name: 'punctuation.definition.link.markdown', + }, + '9': { + name: 'string.other.link.description.title.markdown', + }, + '10': { + name: 'punctuation.definition.string.markdown', + }, + '11': { + name: 'punctuation.definition.string.markdown', + }, + '12': { + name: 'string.other.link.description.title.markdown', + }, + '13': { + name: 'punctuation.definition.string.markdown', + }, + '14': { + name: 'punctuation.definition.string.markdown', + }, + '15': { + name: 'string.other.link.description.title.markdown', + }, + '16': { + name: 'punctuation.definition.string.markdown', + }, + '17': { + name: 'punctuation.definition.string.markdown', + }, + '18': { + name: 'punctuation.definition.metadata.markdown', + }, + }, + match: '(?x)\n (\\!\\[)((?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])*+)(\\])\n # Match the link text.\n (\\() # Opening paren for url\n (<?)(\\S+?)(>?) # The url\n [ \\t]* # Optional whitespace\n (?:\n ((\\().+?(\\))) # Match title in parensโ€ฆ\n | ((").+?(")) # or in double quotesโ€ฆ\n | ((\').+?(\')) # or in single quotes.\n )? # Title is optional\n \\s* # Optional whitespace\n (\\))\n', + name: 'meta.image.inline.markdown', + }, + 'image-ref': { + captures: { + '1': { + name: 'punctuation.definition.link.description.begin.markdown', + }, + '2': { + name: 'string.other.link.description.markdown', + }, + '4': { + name: 'punctuation.definition.link.description.end.markdown', + }, + '5': { + name: 'punctuation.definition.constant.markdown', + }, + '6': { + name: 'constant.other.reference.link.markdown', + }, + '7': { + name: 'punctuation.definition.constant.markdown', + }, + }, + match: '(\\!\\[)((?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])*+)(\\])[ ]?(\\[)(.*?)(\\])', + name: 'meta.image.reference.markdown', + }, + italic: { + begin: '(?x) (?<open>(\\*(?=\\w)|(?<!\\w)\\*|(?<!\\w)\\b_))(?=\\S) # Open\n (?=\n (\n <[^>]*+> # HTML tags\n | (?<raw>`+)([^`]|(?!(?<!`)\\k<raw>(?!`))`)*+\\k<raw>\n # Raw\n | \\\\[\\\\`*_{}\\[\\]()#.!+\\->]?+ # Escapes\n | \\[\n (\n (?<square> # Named group\n [^\\[\\]\\\\] # Match most chars\n | \\\\. # Escaped chars\n | \\[ \\g<square>*+ \\] # Nested brackets\n )*+\n \\]\n (\n ( # Reference Link\n [ ]? # Optional space\n \\[[^\\]]*+\\] # Ref name\n )\n | ( # Inline Link\n \\( # Opening paren\n [ \\t]*+ # Optional whtiespace\n <?(.*?)>? # URL\n [ \\t]*+ # Optional whtiespace\n ( # Optional Title\n (?<title>[\'"])\n (.*?)\n \\k<title>\n )?\n \\)\n )\n )\n )\n | \\k<open>\\k<open> # Must be bold closer\n | (?!(?<=\\S)\\k<open>). # Everything besides\n # style closer\n )++\n (?<=\\S)(?=_\\b|\\*)\\k<open> # Close\n )\n', + captures: { + '1': { + name: 'punctuation.definition.italic.markdown', + }, + }, + end: '(?<=\\S)(\\1)((?!\\1)|(?=\\1\\1))', + name: 'markup.italic.markdown', + patterns: [ + { + applyEndPatternLast: true, + begin: '(?=<[^>]*?>)', + end: '(?<=>)', + patterns: [ + { + include: 'text.html.derivative', + }, + ], + }, + { + include: '#escape', + }, + { + include: '#ampersand', + }, + { + include: '#bracket', + }, + { + include: '#raw', + }, + { + include: '#bold', + }, + { + include: '#image-inline', + }, + { + include: '#link-inline', + }, + { + include: '#link-inet', + }, + { + include: '#link-email', + }, + { + include: '#image-ref', + }, + { + include: '#link-ref-literal', + }, + { + include: '#link-ref', + }, + { + include: '#link-ref-shortcut', + }, + { + include: '#strikethrough', + }, + ], + }, + 'link-email': { + captures: { + '1': { + name: 'punctuation.definition.link.markdown', + }, + '2': { + name: 'markup.underline.link.markdown', + }, + '4': { + name: 'punctuation.definition.link.markdown', + }, + }, + match: "(<)((?:mailto:)?[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\\.[a-zA-Z0-9-]+)*)(>)", + name: 'meta.link.email.lt-gt.markdown', + }, + 'link-inet': { + captures: { + '1': { + name: 'punctuation.definition.link.markdown', + }, + '2': { + name: 'markup.underline.link.markdown', + }, + '3': { + name: 'punctuation.definition.link.markdown', + }, + }, + match: '(<)((?:https?|ftp)://.*?)(>)', + name: 'meta.link.inet.markdown', + }, + 'link-inline': { + captures: { + '1': { + name: 'punctuation.definition.link.title.begin.markdown', + }, + '2': { + name: 'string.other.link.title.markdown', + patterns: [ + { + include: '#raw', + }, + { + include: '#bold', + }, + { + include: '#italic', + }, + { + include: '#strikethrough', + }, + { + include: '#image-inline', + }, + ], + }, + '4': { + name: 'punctuation.definition.link.title.end.markdown', + }, + '5': { + name: 'punctuation.definition.metadata.markdown', + }, + '7': { + name: 'punctuation.definition.link.markdown', + }, + '8': { + name: 'markup.underline.link.markdown', + }, + '9': { + name: 'punctuation.definition.link.markdown', + }, + '10': { + name: 'markup.underline.link.markdown', + }, + '12': { + name: 'string.other.link.description.title.markdown', + }, + '13': { + name: 'punctuation.definition.string.begin.markdown', + }, + '14': { + name: 'punctuation.definition.string.end.markdown', + }, + '15': { + name: 'string.other.link.description.title.markdown', + }, + '16': { + name: 'punctuation.definition.string.begin.markdown', + }, + '17': { + name: 'punctuation.definition.string.end.markdown', + }, + '18': { + name: 'string.other.link.description.title.markdown', + }, + '19': { + name: 'punctuation.definition.string.begin.markdown', + }, + '20': { + name: 'punctuation.definition.string.end.markdown', + }, + '21': { + name: 'punctuation.definition.metadata.markdown', + }, + }, + match: '(?x)\n (\\[)((?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])*+)(\\])\n # Match the link text.\n (\\() # Opening paren for url\n # The url\n [ \\t]*\n (\n (<)([^<>\\n]*)(>)\n | ((?<url>(?>[^\\s()]+)|\\(\\g<url>*\\))*)\n )\n [ \\t]*\n # The title \n (?:\n ((\\()[^()]*(\\))) # Match title in parensโ€ฆ\n | ((")[^"]*(")) # or in double quotesโ€ฆ\n | ((\')[^\']*(\')) # or in single quotes.\n )? # Title is optional\n \\s* # Optional whitespace\n (\\))\n', + name: 'meta.link.inline.markdown', + }, + 'link-ref': { + captures: { + '1': { + name: 'punctuation.definition.link.title.begin.markdown', + }, + '2': { + name: 'string.other.link.title.markdown', + patterns: [ + { + include: '#raw', + }, + { + include: '#bold', + }, + { + include: '#italic', + }, + { + include: '#strikethrough', + }, + { + include: '#image-inline', + }, + ], + }, + '4': { + name: 'punctuation.definition.link.title.end.markdown', + }, + '5': { + name: 'punctuation.definition.constant.begin.markdown', + }, + '6': { + name: 'constant.other.reference.link.markdown', + }, + '7': { + name: 'punctuation.definition.constant.end.markdown', + }, + }, + match: '(?<![\\]\\\\])(\\[)((?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])*+)(\\])(\\[)([^\\]]*+)(\\])', + name: 'meta.link.reference.markdown', + }, + 'link-ref-literal': { + captures: { + '1': { + name: 'punctuation.definition.link.title.begin.markdown', + }, + '2': { + name: 'string.other.link.title.markdown', + }, + '4': { + name: 'punctuation.definition.link.title.end.markdown', + }, + '5': { + name: 'punctuation.definition.constant.begin.markdown', + }, + '6': { + name: 'punctuation.definition.constant.end.markdown', + }, + }, + match: '(?<![\\]\\\\])(\\[)((?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])*+)(\\])[ ]?(\\[)(\\])', + name: 'meta.link.reference.literal.markdown', + }, + 'link-ref-shortcut': { + captures: { + '1': { + name: 'punctuation.definition.link.title.begin.markdown', + }, + '2': { + name: 'string.other.link.title.markdown', + }, + '3': { + name: 'punctuation.definition.link.title.end.markdown', + }, + }, + match: '(?<![\\]\\\\])(\\[)(\\S+?)(\\])', + name: 'meta.link.reference.markdown', + }, + raw: { + captures: { + '1': { + name: 'punctuation.definition.raw.markdown', + }, + '3': { + name: 'punctuation.definition.raw.markdown', + }, + }, + match: '(`+)((?:[^`]|(?!(?<!`)\\1(?!`))`)*+)(\\1)', + name: 'markup.inline.raw.string.markdown', + }, + strikethrough: { + captures: { + '1': { + name: 'punctuation.definition.strikethrough.markdown', + }, + '2': { + patterns: [ + { + applyEndPatternLast: true, + begin: '(?=<[^>]*?>)', + end: '(?<=>)', + patterns: [ + { + include: 'text.html.derivative', + }, + ], + }, + { + include: '#escape', + }, + { + include: '#ampersand', + }, + { + include: '#bracket', + }, + { + include: '#raw', + }, + { + include: '#bold', + }, + { + include: '#italic', + }, + { + include: '#image-inline', + }, + { + include: '#link-inline', + }, + { + include: '#link-inet', + }, + { + include: '#link-email', + }, + { + include: '#image-ref', + }, + { + include: '#link-ref-literal', + }, + { + include: '#link-ref', + }, + { + include: '#link-ref-shortcut', + }, + ], + }, + '3': { + name: 'punctuation.definition.strikethrough.markdown', + }, + }, + match: '(?<!\\\\)(~{2,})((?:[^~]|(?!(?<![~\\\\])\\1(?!~))~)*+)(\\1)', + name: 'markup.strikethrough.markdown', + }, + }, +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/languages/md-math.tmLanguage.ts b/completions-sample-code/vscode-node/extension/src/panelShared/languages/md-math.tmLanguage.ts new file mode 100644 index 0000000..acc6c82 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/languages/md-math.tmLanguage.ts @@ -0,0 +1,113 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { LanguageInput } from 'shiki/core'; + +// This file includes some grammar rules copied from https://github.com/James-Yu/LaTeX-Workshop/blob/master/syntax/TeX.tmLanguage.json', +export const markdownMath: LanguageInput = { + name: 'markdown-math', + scopeName: 'text.html.markdown.math', + patterns: [ + { + include: '#math', + }, + ], + repository: { + $self: {}, + $base: {}, + math: { + patterns: [ + { + name: 'comment.line.math.tex', + match: '((?<!\\\\)%)(.+)$', + captures: { + '1': { + name: 'punctuation.definition.comment.math.tex', + }, + }, + }, + { + name: 'line.separator.math.tex', + match: '(\\\\\\\\)$', + captures: { + '1': { + name: 'punctuation.line.separator.math.tex', + }, + }, + }, + { + name: 'meta.function.math.tex', + begin: '((\\\\)([a-zA-Z_]+))\\s*(\\{)', + beginCaptures: { + '1': { + name: 'storage.type.function.math.tex', + }, + '2': { + name: 'punctuation.definition.function.math.tex', + }, + '3': { + name: 'entity.name.function.math.tex', + }, + '4': { + name: 'punctuation.definition.arguments.begin.math.tex', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.arguments.end.math.tex', + }, + }, + patterns: [ + { + include: '$self', + }, + ], + }, + { + captures: { + '1': { + name: 'punctuation.definition.constant.math.tex', + }, + }, + match: '(\\\\)([a-zA-Z_]+)\\b', + name: 'constant.character.math.tex', + }, + { + captures: { + '1': { + name: 'punctuation.definition.constant.math.tex', + }, + }, + match: '(\\\\)(?!begin\\*\\{|verb)([A-Za-z]+)', + name: 'constant.other.general.math.tex', + }, + { + match: '(?<!\\\\)\\{', + name: 'punctuation.math.begin.bracket.curly', + }, + { + match: '(?<!\\\\)\\}', + name: 'punctuation.math.end.bracket.curly', + }, + { + match: '\\(', + name: 'punctuation.math.begin.bracket.round', + }, + { + match: '\\)', + name: 'punctuation.math.end.bracket.round', + }, + { + match: '(([0-9]*[\\.][0-9]+)|[0-9]+)', + name: 'constant.numeric.math.tex', + }, + { + match: '[\\+\\*/_\\^-]', + name: 'punctuation.math.operator.latex', + }, + ], + }, + }, +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/languages/rst.tmLanguage.ts b/completions-sample-code/vscode-node/extension/src/panelShared/languages/rst.tmLanguage.ts new file mode 100644 index 0000000..999bb5e --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/languages/rst.tmLanguage.ts @@ -0,0 +1,740 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/* eslint-disable local/no-unexternalized-strings */ +import { LanguageInput } from 'shiki/core'; + +// This file has been converted from https://github.com/trond-snekvik/vscode-rst/blob/master/syntaxes/rst.tmLanguage.json +// If you want to provide a fix or improvement, please create a pull request against the original repository. +// Once accepted there, we are happy to receive an update request. +// version: https://github.com/trond-snekvik/vscode-rst/commit/f0fe19ffde6509be52ad9267a57e1b3df665f072 +export const restructuredtext: LanguageInput = { + scopeName: 'source.rst', + name: 'restructuredtext', + patterns: [ + { + include: '#body', + }, + ], + repository: { + $self: {}, + $base: {}, + body: { + patterns: [ + { + include: '#title', + }, + { + include: '#inline-markup', + }, + { + include: '#anchor', + }, + { + include: '#line-block', + }, + { + include: '#replace-include', + }, + { + include: '#footnote', + }, + { + include: '#substitution', + }, + { + include: '#blocks', + }, + { + include: '#table', + }, + { + include: '#simple-table', + }, + { + include: '#options-list', + }, + ], + }, + title: { + match: '^(\\*{3,}|#{3,}|\\={3,}|~{3,}|\\+{3,}|-{3,}|`{3,}|\\^{3,}|:{3,}|"{3,}|_{3,}|\'{3,})$', + name: 'markup.heading', + }, + 'inline-markup': { + patterns: [ + { + include: '#escaped', + }, + { + include: '#ignore', + }, + { + include: '#ref', + }, + { + include: '#literal', + }, + { + include: '#monospaced', + }, + { + include: '#citation', + }, + { + include: '#bold', + }, + { + include: '#italic', + }, + { + include: '#list', + }, + { + include: '#macro', + }, + { + include: '#reference', + }, + { + include: '#footnote-ref', + }, + ], + }, + ignore: { + patterns: [ + { + match: "'[`*]+'", + }, + { + match: '<[`*]+>', + }, + { + match: '{[`*]+}', + }, + { + match: '\\([`*]+\\)', + }, + { + match: '\\[[`*]+\\]', + }, + { + match: '"[`*]+"', + }, + ], + }, + table: { + begin: '^\\s*\\+[=+-]+\\+\\s*$', + end: '^(?![+|])', + beginCaptures: { + '0': { + name: 'keyword.control.table', + }, + }, + patterns: [ + { + match: '[=+|-]', + name: 'keyword.control.table', + }, + ], + }, + 'simple-table': { + match: '^[=\\s]+$', + name: 'keyword.control.table', + }, + ref: { + begin: '(:ref:)`', + end: '`|^\\s*$', + name: 'entity.name.tag', + beginCaptures: { + '1': { + name: 'keyword.control', + }, + }, + patterns: [ + { + match: '<.*?>', + name: 'markup.underline.link', + }, + ], + }, + reference: { + match: '[\\w-]*[a-zA-Z\\d-]__?\\b', + name: 'entity.name.tag', + }, + macro: { + match: '\\|[^\\|]+\\|', + name: 'entity.name.tag', + }, + literal: { + match: '(:\\S+:)(`.*?`\\\\?)', + captures: { + '1': { + name: 'keyword.control', + }, + '2': { + name: 'entity.name.tag', + }, + }, + }, + monospaced: { + begin: '(?<=[\\s"\'(\\[{<]|^)``[^\\s`]', + end: '``|^\\s*$', + name: 'string.interpolated', + }, + citation: { + begin: '(?<=[\\s"\'(\\[{<]|^)`[^\\s`]', + end: '`_{,2}|^\\s*$', + name: 'entity.name.tag', + applyEndPatternLast: false, + }, + bold: { + begin: '(?<=[\\s"\'(\\[{<]|^)\\*{2}[^\\s*]', + end: '\\*{2}|^\\s*$', + name: 'markup.bold', + }, + italic: { + begin: '(?<=[\\s"\'(\\[{<]|^)\\*[^\\s*]', + end: '\\*|^\\s*$', + name: 'markup.italic', + }, + escaped: { + match: '\\\\.', + name: 'constant.character.escape', + }, + list: { + match: '^\\s*(\\d+\\.|\\* -|[a-zA-Z#]\\.|[iIvVxXmMcC]+\\.|\\(\\d+\\)|\\d+\\)|[*+-])\\s+', + name: 'keyword.control', + }, + 'line-block': { + match: '^\\|\\s+', + name: 'keyword.control', + }, + 'raw-html': { + begin: '^(\\s*)(\\.{2}\\s+raw\\s*::)\\s+(html)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '3': { + name: 'variable.parameter.html', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'text.html.derivative', + }, + ], + }, + anchor: { + match: '^\\.{2}\\s+(_[^:]+:)\\s*', + name: 'entity.name.tag.anchor', + }, + 'replace-include': { + match: '^\\s*(\\.{2})\\s+(\\|[^\\|]+\\|)\\s+(replace::)', + captures: { + '1': { + name: 'keyword.control', + }, + '2': { + name: 'entity.name.tag', + }, + '3': { + name: 'keyword.control', + }, + }, + }, + footnote: { + match: '^\\s*\\.{2}\\s+\\[(?:[\\w\\.-]+|[#*]|#\\w+)\\]\\s+', + name: 'entity.name.tag', + }, + 'footnote-ref': { + match: '\\[(?:[\\w\\.-]+|[#*])\\]_', + name: 'entity.name.tag', + }, + substitution: { + match: '^\\.{2}\\s*\\|([^|]+)\\|', + name: 'entity.name.tag', + }, + 'options-list': { + match: '^((?:-\\w|--[\\w-]+|/\\w+)(?:,? ?[\\w-]+)*)(?: |\\t|$)', + name: 'variable.parameter', + }, + blocks: { + patterns: [ + { + include: '#domains', + }, + { + include: '#doctest', + }, + { + include: '#code-block-cpp', + }, + { + include: '#code-block-py', + }, + { + include: '#code-block-console', + }, + { + include: '#code-block-javascript', + }, + { + include: '#code-block-yaml', + }, + { + include: '#code-block-cmake', + }, + { + include: '#code-block-kconfig', + }, + { + include: '#code-block-ruby', + }, + { + include: '#code-block-dts', + }, + { + include: '#code-block', + }, + { + include: '#doctest-block', + }, + { + include: '#raw-html', + }, + { + include: '#block', + }, + { + include: '#literal-block', + }, + { + include: '#block-comment', + }, + ], + }, + 'block-comment': { + begin: '^(\\s*)\\.{2}', + while: '^\\1(?=\\s)|^\\s*$', + name: 'comment.block', + }, + 'literal-block': { + begin: '^(\\s*)(.*)(::)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + patterns: [ + { + include: '#inline-markup', + }, + ], + }, + '3': { + name: 'keyword.control', + }, + }, + }, + block: { + begin: '^(\\s*)(\\.{2}\\s+\\S+::)(.*)', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '3': { + name: 'variable', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: '#body', + }, + ], + }, + 'block-param': { + patterns: [ + { + match: '(:param\\s+(.+?):)(?:\\s|$)', + captures: { + '1': { + name: 'keyword.control', + }, + '2': { + name: 'variable.parameter', + }, + }, + }, + { + match: '(:.+?:)(?:$|\\s+(.*))', + captures: { + '1': { + name: 'keyword.control', + }, + '2': { + patterns: [ + { + match: '\\b(0x[a-fA-F\\d]+|\\d+)\\b', + name: 'constant.numeric', + }, + { + include: '#inline-markup', + }, + ], + }, + }, + }, + ], + }, + domains: { + patterns: [ + { + include: '#domain-cpp', + }, + { + include: '#domain-py', + }, + { + include: '#domain-auto', + }, + { + include: '#domain-js', + }, + ], + }, + 'domain-cpp': { + begin: '^(\\s*)(\\.{2}\\s+(?:cpp|c):(?:class|struct|function|member|var|type|enum|enum-struct|enum-class|enumerator|union|concept)::)\\s*(?:(@\\w+)|(.*))', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '3': { + name: 'entity.name.tag', + }, + '4': { + patterns: [ + { + include: 'source.cpp', + }, + ], + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: '#body', + }, + ], + }, + 'domain-py': { + begin: '^(\\s*)(\\.{2}\\s+py:(?:module|function|data|exception|class|attribute|property|method|staticmethod|classmethod|decorator|decoratormethod)::)\\s*(.*)', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '3': { + patterns: [ + { + include: 'source.python', + }, + ], + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: '#body', + }, + ], + }, + 'domain-auto': { + begin: '^(\\s*)(\\.{2}\\s+auto(?:class|module|exception|function|decorator|data|method|attribute|property)::)\\s*(.*)', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control.py', + }, + '3': { + patterns: [ + { + include: 'source.python', + }, + ], + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: '#body', + }, + ], + }, + 'domain-js': { + begin: '^(\\s*)(\\.{2}\\s+js:\\w+::)\\s*(.*)', + end: '^(?!\\1[ \\t]|$)', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '3': { + patterns: [ + { + include: 'source.js', + }, + ], + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: '#body', + }, + ], + }, + doctest: { + begin: '^(>>>)\\s*(.*)', + end: '^\\s*$', + beginCaptures: { + '1': { + name: 'keyword.control', + }, + '2': { + patterns: [ + { + include: 'source.python', + }, + ], + }, + }, + }, + 'code-block-cpp': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(c|c\\+\\+|cpp|C|C\\+\\+|CPP|Cpp)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.cpp', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.cpp', + }, + ], + }, + 'code-block-console': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(console|shell|bash)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.console', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.shell', + }, + ], + }, + 'code-block-py': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(python)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.py', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.python', + }, + ], + }, + 'code-block-javascript': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(javascript)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.js', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.js', + }, + ], + }, + 'code-block-yaml': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(ya?ml)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.yaml', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.yaml', + }, + ], + }, + 'code-block-cmake': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(cmake)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.cmake', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.cmake', + }, + ], + }, + 'code-block-kconfig': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*([kK]config)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.kconfig', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.kconfig', + }, + ], + }, + 'code-block-ruby': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(ruby)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.ruby', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.ruby', + }, + ], + }, + 'code-block-dts': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)\\s*(dts|DTS|devicetree)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + '4': { + name: 'variable.parameter.codeblock.dts', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.dts', + }, + ], + }, + 'code-block': { + begin: '^(\\s*)(\\.{2}\\s+(code|code-block)::)', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + }, + patterns: [ + { + include: '#block-param', + }, + ], + }, + 'doctest-block': { + begin: '^(\\s*)(\\.{2}\\s+doctest::)\\s*$', + while: '^\\1(?=\\s)|^\\s*$', + beginCaptures: { + '2': { + name: 'keyword.control', + }, + }, + patterns: [ + { + include: '#block-param', + }, + { + include: 'source.python', + }, + ], + }, + }, +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/languages/searchResult.tmLanguage.ts b/completions-sample-code/vscode-node/extension/src/panelShared/languages/searchResult.tmLanguage.ts new file mode 100644 index 0000000..fd1cfd5 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/languages/searchResult.tmLanguage.ts @@ -0,0 +1,4671 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { LanguageInput } from 'shiki/core'; + +// This file is generated from ./generateTMLanguage.js. +export const searchResult: LanguageInput = { + name: 'search-result', + scopeName: 'text.searchResult', + patterns: [ + { + begin: '^(# Query): ', + end: '\n', + name: 'meta.header.search keyword.operator.word.search', + beginCaptures: { + '1': { + name: 'entity.other.attribute-name', + }, + }, + patterns: [ + { + match: '(\\\\n)|(\\\\\\\\)', + name: 'entity.other.attribute-value string.unquoted constant.character.escape', + }, + { + match: '\\\\.|\\\\$', + name: 'entity.other.attribute-value string.unquoted invalid.illegal', + }, + { + match: '[^\\\\\\\n]+', + name: 'entity.other.attribute-value string.unquoted', + }, + ], + }, + { + begin: '^(# Flags): ', + end: '\n', + name: 'meta.header.search keyword.operator.word.search', + beginCaptures: { + '1': { + name: 'entity.other.attribute-name', + }, + }, + patterns: [ + { + match: '(RegExp|CaseSensitive|IgnoreExcludeSettings|WordMatch)', + name: 'entity.other.attribute-value string.unquoted keyword.other', + }, + { + match: '.', + }, + ], + }, + { + begin: '^(# ContextLines): ', + end: '\n', + name: 'meta.header.search keyword.operator.word.search', + beginCaptures: { + '1': { + name: 'entity.other.attribute-name', + }, + }, + patterns: [ + { + match: '\\d', + name: 'entity.other.attribute-value string.unquoted constant.numeric.integer', + }, + { + match: '.', + name: 'invalid.illegal', + }, + ], + }, + { + match: '^(# (?:Including|Excluding)): (.*)$', + name: 'meta.header.search keyword.operator.word.search', + captures: { + '1': { + name: 'entity.other.attribute-name', + }, + '2': { + name: 'entity.other.attribute-value string.unquoted', + }, + }, + }, + { + include: '#bat', + }, + { + include: '#c', + }, + { + include: '#clj', + }, + { + include: '#coffee', + }, + { + include: '#cpp', + }, + { + include: '#cs', + }, + { + include: '#cshtml', + }, + { + include: '#css', + }, + { + include: '#dart', + }, + { + include: '#diff', + }, + { + include: '#dockerfile', + }, + { + include: '#fs', + }, + { + include: '#go', + }, + { + include: '#groovy', + }, + { + include: '#h', + }, + { + include: '#handlebars', + }, + { + include: '#hlsl', + }, + { + include: '#hpp', + }, + { + include: '#html', + }, + { + include: '#ini', + }, + { + include: '#java', + }, + { + include: '#jl', + }, + { + include: '#js', + }, + { + include: '#json', + }, + { + include: '#jsx', + }, + { + include: '#less', + }, + { + include: '#log', + }, + { + include: '#lua', + }, + { + include: '#m', + }, + { + include: '#makefile', + }, + { + include: '#md', + }, + { + include: '#mm', + }, + { + include: '#p6', + }, + { + include: '#perl', + }, + { + include: '#php', + }, + { + include: '#ps1', + }, + { + include: '#pug', + }, + { + include: '#py', + }, + { + include: '#r', + }, + { + include: '#rb', + }, + { + include: '#rs', + }, + { + include: '#scala', + }, + { + include: '#scss', + }, + { + include: '#sh', + }, + { + include: '#sql', + }, + { + include: '#swift', + }, + { + include: '#ts', + }, + { + include: '#tsx', + }, + { + include: '#vb', + }, + { + include: '#xml', + }, + { + include: '#yaml', + }, + { + match: '^(?!\\s)(.*?)([^\\\\\\/\\n]*)(:)$', + name: 'meta.resultBlock.search string meta.path.search', + captures: { + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + }, + { + match: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+)( ))(.*))', + name: 'meta.resultBlock.search meta.resultLine.search', + captures: { + '1': { + name: 'constant.numeric.integer meta.resultLinePrefix.search meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'constant.numeric.integer meta.resultLinePrefix.search meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + }, + { + match: 'โŸช [0-9]+ characters skipped โŸซ', + name: 'meta.resultBlock.search comment meta.resultLine.elision', + }, + ], + repository: { + $self: {}, + $base: {}, + bat: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.bat)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.batchfile', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.batchfile', + }, + ], + }, + ], + }, + c: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.c)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.c', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.c', + }, + ], + }, + ], + }, + clj: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.clj)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.clojure', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.clojure', + }, + ], + }, + ], + }, + coffee: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.coffee)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.coffee', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.coffee', + }, + ], + }, + ], + }, + cpp: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.(?:cpp|c\\+\\+|cc|cxx|hxx|h\\+\\+|hh))(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.cpp', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.cpp', + }, + ], + }, + ], + }, + cs: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.cs)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.cs', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.cs', + }, + ], + }, + ], + }, + cshtml: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.cshtml)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.html.cshtml', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.html.cshtml', + }, + ], + }, + ], + }, + css: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.css)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.css', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.css', + }, + ], + }, + ], + }, + dart: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.dart)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.dart', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.dart', + }, + ], + }, + ], + }, + diff: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.diff)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.diff', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.diff', + }, + ], + }, + ], + }, + dockerfile: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*(?:dockerfile|Dockerfile|containerfile|Containerfile))(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.dockerfile', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.dockerfile', + }, + ], + }, + ], + }, + fs: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.fs)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.fsharp', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.fsharp', + }, + ], + }, + ], + }, + go: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.go)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.go', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.go', + }, + ], + }, + ], + }, + groovy: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.groovy)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.groovy', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.groovy', + }, + ], + }, + ], + }, + h: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.h)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.objc', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.objc', + }, + ], + }, + ], + }, + handlebars: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.(?:handlebars|hbs))(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.html.handlebars', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.html.handlebars', + }, + ], + }, + ], + }, + hlsl: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.hlsl)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.hlsl', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.hlsl', + }, + ], + }, + ], + }, + hpp: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.hpp)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.objcpp', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.objcpp', + }, + ], + }, + ], + }, + html: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.html)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.html.basic', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.html.basic', + }, + ], + }, + ], + }, + ini: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.ini)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.ini', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.ini', + }, + ], + }, + ], + }, + java: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.java)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.java', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.java', + }, + ], + }, + ], + }, + jl: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.jl)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.julia', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.julia', + }, + ], + }, + ], + }, + js: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.js)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.js', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.js', + }, + ], + }, + ], + }, + json: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.json)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.json.comments', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.json.comments', + }, + ], + }, + ], + }, + jsx: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.jsx)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.js.jsx', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.js.jsx', + }, + ], + }, + ], + }, + less: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.less)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.css.less', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.css.less', + }, + ], + }, + ], + }, + log: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.log)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.log', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.log', + }, + ], + }, + ], + }, + lua: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.lua)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.lua', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.lua', + }, + ], + }, + ], + }, + m: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.m)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.objc', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.objc', + }, + ], + }, + ], + }, + makefile: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*(?:makefile|Makefile)(?:\\..*)?)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.makefile', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.makefile', + }, + ], + }, + ], + }, + md: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.md)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.html.markdown', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.html.markdown', + }, + ], + }, + ], + }, + mm: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.mm)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.objcpp', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.objcpp', + }, + ], + }, + ], + }, + p6: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.p6)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.perl.6', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.perl.6', + }, + ], + }, + ], + }, + perl: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.(?:perl|pl|pm))(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.perl', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.perl', + }, + ], + }, + ], + }, + php: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.php)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.php', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.php', + }, + ], + }, + ], + }, + ps1: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.ps1)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.powershell', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.powershell', + }, + ], + }, + ], + }, + pug: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.pug)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.pug', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.pug', + }, + ], + }, + ], + }, + py: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.py)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.python', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.python', + }, + ], + }, + ], + }, + r: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.r)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.r', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.r', + }, + ], + }, + ], + }, + rb: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.rb)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.ruby', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.ruby', + }, + ], + }, + ], + }, + rs: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.rs)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.rust', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.rust', + }, + ], + }, + ], + }, + scala: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.scala)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.scala', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.scala', + }, + ], + }, + ], + }, + scss: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.scss)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.css.scss', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.css.scss', + }, + ], + }, + ], + }, + sh: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.sh)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.shell', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.shell', + }, + ], + }, + ], + }, + sql: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.sql)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.sql', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.sql', + }, + ], + }, + ], + }, + swift: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.swift)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.swift', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.swift', + }, + ], + }, + ], + }, + ts: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.ts)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.ts', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.ts', + }, + ], + }, + ], + }, + tsx: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.tsx)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.tsx', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.tsx', + }, + ], + }, + ], + }, + vb: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.vb)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.asp.vb.net', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.asp.vb.net', + }, + ], + }, + ], + }, + xml: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.xml)(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'text.xml', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'text.xml', + }, + ], + }, + ], + }, + yaml: { + name: 'meta.resultBlock.search', + begin: '^(?!\\s)(.*?)([^\\\\\\/\\n]*\\.(?:ya?ml))(:)$', + end: '^(?!\\s)', + beginCaptures: { + '0': { + name: 'string meta.path.search', + }, + '1': { + name: 'meta.path.dirname.search', + }, + '2': { + name: 'meta.path.basename.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + name: 'meta.resultLine.search meta.resultLine.multiLine.search', + begin: '^ (?:\\s*)((\\d+) )', + while: '^ (?:\\s*)(?:((\\d+)(:))|((\\d+) ))', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + whileCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + '4': { + name: 'meta.resultLinePrefix.contextLinePrefix.search', + }, + '5': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + }, + patterns: [ + { + include: 'source.yaml', + }, + ], + }, + { + begin: '^ (?:\\s*)((\\d+)(:))', + while: '(?=not)possible', + name: 'meta.resultLine.search meta.resultLine.singleLine.search', + beginCaptures: { + '0': { + name: 'constant.numeric.integer meta.resultLinePrefix.search', + }, + '1': { + name: 'meta.resultLinePrefix.matchLinePrefix.search', + }, + '2': { + name: 'meta.resultLinePrefix.lineNumber.search', + }, + '3': { + name: 'punctuation.separator', + }, + }, + patterns: [ + { + include: 'source.yaml', + }, + ], + }, + ], + }, + }, +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/languages/typeScriptReact.tmLanguage.ts b/completions-sample-code/vscode-node/extension/src/panelShared/languages/typeScriptReact.tmLanguage.ts new file mode 100644 index 0000000..7d7968e --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/languages/typeScriptReact.tmLanguage.ts @@ -0,0 +1,5927 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { LanguageInput } from 'shiki/core'; + +// This file has been converted from https://github.com/microsoft/TypeScript-TmLanguage/blob/master/TypeScriptReact.tmLanguage +// If you want to provide a fix or improvement, please create a pull request against the original repository. +// Once accepted there, we are happy to receive an update request. +// version: https://github.com/microsoft/TypeScript-TmLanguage/commit/0d73d1117e0a9b1d6635ebbe9aa37d615171b02d +export const typescriptreact: LanguageInput = { + name: 'typescriptreact', + scopeName: 'source.tsx', + patterns: [ + { + include: '#directives', + }, + { + include: '#statements', + }, + { + include: '#shebang', + }, + ], + repository: { + $self: {}, + $base: {}, + shebang: { + name: 'comment.line.shebang.tsx', + match: '\\A(#!).*(?=$)', + captures: { + '1': { + name: 'punctuation.definition.comment.tsx', + }, + }, + }, + statements: { + patterns: [ + { + include: '#declaration', + }, + { + include: '#control-statement', + }, + { + include: '#after-operator-block-as-object-literal', + }, + { + include: '#decl-block', + }, + { + include: '#label', + }, + { + include: '#expression', + }, + { + include: '#punctuation-semicolon', + }, + { + include: '#string', + }, + { + include: '#comment', + }, + ], + }, + declaration: { + patterns: [ + { + include: '#decorator', + }, + { + include: '#var-expr', + }, + { + include: '#function-declaration', + }, + { + include: '#class-declaration', + }, + { + include: '#interface-declaration', + }, + { + include: '#enum-declaration', + }, + { + include: '#namespace-declaration', + }, + { + include: '#type-alias-declaration', + }, + { + include: '#import-equals-declaration', + }, + { + include: '#import-declaration', + }, + { + include: '#export-declaration', + }, + { + name: 'storage.modifier.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(declare|export)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + ], + }, + 'control-statement': { + patterns: [ + { + include: '#switch-statement', + }, + { + include: '#for-loop', + }, + { + name: 'keyword.control.trycatch.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(catch|finally|throw|try)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(break|continue|goto)\\s+([_$[:alpha:]][_$[:alnum:]]*)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + captures: { + '1': { + name: 'keyword.control.loop.tsx', + }, + '2': { + name: 'entity.name.label.tsx', + }, + }, + }, + { + name: 'keyword.control.loop.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(break|continue|do|goto|while)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(return)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '0': { + name: 'keyword.control.flow.tsx', + }, + }, + end: '(?=[;}]|$|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + name: 'keyword.control.switch.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(case|default|switch)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + include: '#if-statement', + }, + { + name: 'keyword.control.conditional.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(else|if)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.control.with.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(with)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.control.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(package)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.other.debugger.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(debugger)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + ], + }, + label: { + patterns: [ + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(:)(?=\\s*\\{)', + beginCaptures: { + '1': { + name: 'entity.name.label.tsx', + }, + '2': { + name: 'punctuation.separator.label.tsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#decl-block', + }, + ], + }, + { + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(:)', + captures: { + '1': { + name: 'entity.name.label.tsx', + }, + '2': { + name: 'punctuation.separator.label.tsx', + }, + }, + }, + ], + }, + expression: { + patterns: [ + { + include: '#expressionWithoutIdentifiers', + }, + { + include: '#identifiers', + }, + { + include: '#expressionPunctuations', + }, + ], + }, + expressionWithoutIdentifiers: { + patterns: [ + { + include: '#jsx', + }, + { + include: '#string', + }, + { + include: '#regex', + }, + { + include: '#comment', + }, + { + include: '#function-expression', + }, + { + include: '#class-expression', + }, + { + include: '#arrow-function', + }, + { + include: '#paren-expression-possibly-arrow', + }, + { + include: '#cast', + }, + { + include: '#ternary-expression', + }, + { + include: '#new-expr', + }, + { + include: '#instanceof-expr', + }, + { + include: '#object-literal', + }, + { + include: '#expression-operators', + }, + { + include: '#function-call', + }, + { + include: '#literal', + }, + { + include: '#support-objects', + }, + { + include: '#paren-expression', + }, + ], + }, + expressionPunctuations: { + patterns: [ + { + include: '#punctuation-comma', + }, + { + include: '#punctuation-accessor', + }, + ], + }, + decorator: { + name: 'meta.decorator.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))\\@', + beginCaptures: { + '0': { + name: 'punctuation.decorator.tsx', + }, + }, + end: '(?=\\s)', + patterns: [ + { + include: '#expression', + }, + ], + }, + 'var-expr': { + patterns: [ + { + name: 'meta.var.expr.tsx', + begin: '(?=(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(var|let)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))', + end: '(?!(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(var|let)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))((?=;|}|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+)|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))|((?<!^let|[^\\._$[:alnum:]]let|^var|[^\\._$[:alnum:]]var)(?=\\s*$)))', + patterns: [ + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(var|let)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.type.tsx', + }, + }, + end: '(?=\\S)', + }, + { + include: '#destructuring-variable', + }, + { + include: '#var-single-variable', + }, + { + include: '#variable-initializer', + }, + { + include: '#comment', + }, + { + begin: '(,)\\s*((?!\\S)|(?=\\/\\/))', + beginCaptures: { + '1': { + name: 'punctuation.separator.comma.tsx', + }, + }, + end: '(?<!,)(((?==|;|}|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+)|^\\s*$))|((?<=\\S)(?=\\s*$)))', + patterns: [ + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#comment', + }, + { + include: '#destructuring-variable', + }, + { + include: '#var-single-variable', + }, + { + include: '#punctuation-comma', + }, + ], + }, + { + include: '#punctuation-comma', + }, + ], + }, + { + name: 'meta.var.expr.tsx', + begin: '(?=(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(const(?!\\s+enum\\b))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.type.tsx', + }, + }, + end: '(?!(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(const(?!\\s+enum\\b))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))((?=;|}|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+)|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))|((?<!^const|[^\\._$[:alnum:]]const)(?=\\s*$)))', + patterns: [ + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(const(?!\\s+enum\\b))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.type.tsx', + }, + }, + end: '(?=\\S)', + }, + { + include: '#destructuring-const', + }, + { + include: '#var-single-const', + }, + { + include: '#variable-initializer', + }, + { + include: '#comment', + }, + { + begin: '(,)\\s*((?!\\S)|(?=\\/\\/))', + beginCaptures: { + '1': { + name: 'punctuation.separator.comma.tsx', + }, + }, + end: '(?<!,)(((?==|;|}|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+)|^\\s*$))|((?<=\\S)(?=\\s*$)))', + patterns: [ + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#comment', + }, + { + include: '#destructuring-const', + }, + { + include: '#var-single-const', + }, + { + include: '#punctuation-comma', + }, + ], + }, + { + include: '#punctuation-comma', + }, + ], + }, + ], + }, + 'var-single-variable': { + patterns: [ + { + name: 'meta.var-single-variable.expr.tsx', + begin: '(?x)([_$[:alpha:]][_$[:alnum:]]*)(\\!)?(?=\\s*\n# function assignment |\n(=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Function(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))) |\n(:\\s*((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + beginCaptures: { + '1': { + name: 'meta.definition.variable.tsx entity.name.function.tsx', + }, + '2': { + name: 'keyword.operator.definiteassignment.tsx', + }, + }, + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#var-single-variable-type-annotation', + }, + ], + }, + { + name: 'meta.var-single-variable.expr.tsx', + begin: '([[:upper:]][_$[:digit:][:upper:]]*)(?![_$[:alnum:]])(\\!)?', + beginCaptures: { + '1': { + name: 'meta.definition.variable.tsx variable.other.constant.tsx', + }, + '2': { + name: 'keyword.operator.definiteassignment.tsx', + }, + }, + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#var-single-variable-type-annotation', + }, + ], + }, + { + name: 'meta.var-single-variable.expr.tsx', + begin: '([_$[:alpha:]][_$[:alnum:]]*)(\\!)?', + beginCaptures: { + '1': { + name: 'meta.definition.variable.tsx variable.other.readwrite.tsx', + }, + '2': { + name: 'keyword.operator.definiteassignment.tsx', + }, + }, + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#var-single-variable-type-annotation', + }, + ], + }, + ], + }, + 'var-single-const': { + patterns: [ + { + name: 'meta.var-single-variable.expr.tsx', + begin: '(?x)([_$[:alpha:]][_$[:alnum:]]*)(?=\\s*\n# function assignment |\n(=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Function(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))) |\n(:\\s*((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + beginCaptures: { + '1': { + name: 'meta.definition.variable.tsx variable.other.constant.tsx entity.name.function.tsx', + }, + }, + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#var-single-variable-type-annotation', + }, + ], + }, + { + name: 'meta.var-single-variable.expr.tsx', + begin: '([_$[:alpha:]][_$[:alnum:]]*)', + beginCaptures: { + '1': { + name: 'meta.definition.variable.tsx variable.other.constant.tsx', + }, + }, + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#var-single-variable-type-annotation', + }, + ], + }, + ], + }, + 'var-single-variable-type-annotation': { + patterns: [ + { + include: '#type-annotation', + }, + { + include: '#string', + }, + { + include: '#comment', + }, + ], + }, + 'destructuring-variable': { + patterns: [ + { + name: 'meta.object-binding-pattern-variable.tsx', + begin: '(?<!=|:|^of|[^\\._$[:alnum:]]of|^in|[^\\._$[:alnum:]]in)\\s*(?=\\{)', + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#object-binding-pattern', + }, + { + include: '#type-annotation', + }, + { + include: '#comment', + }, + ], + }, + { + name: 'meta.array-binding-pattern-variable.tsx', + begin: '(?<!=|:|^of|[^\\._$[:alnum:]]of|^in|[^\\._$[:alnum:]]in)\\s*(?=\\[)', + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#array-binding-pattern', + }, + { + include: '#type-annotation', + }, + { + include: '#comment', + }, + ], + }, + ], + }, + 'destructuring-const': { + patterns: [ + { + name: 'meta.object-binding-pattern-variable.tsx', + begin: '(?<!=|:|^of|[^\\._$[:alnum:]]of|^in|[^\\._$[:alnum:]]in)\\s*(?=\\{)', + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#object-binding-pattern-const', + }, + { + include: '#type-annotation', + }, + { + include: '#comment', + }, + ], + }, + { + name: 'meta.array-binding-pattern-variable.tsx', + begin: '(?<!=|:|^of|[^\\._$[:alnum:]]of|^in|[^\\._$[:alnum:]]in)\\s*(?=\\[)', + end: '(?=$|^|[;,=}]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#array-binding-pattern-const', + }, + { + include: '#type-annotation', + }, + { + include: '#comment', + }, + ], + }, + ], + }, + 'object-binding-element': { + patterns: [ + { + include: '#comment', + }, + { + begin: '(?x)(?=((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(:))', + end: '(?=,|\\})', + patterns: [ + { + include: '#object-binding-element-propertyName', + }, + { + include: '#binding-element', + }, + ], + }, + { + include: '#object-binding-pattern', + }, + { + include: '#destructuring-variable-rest', + }, + { + include: '#variable-initializer', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'object-binding-element-const': { + patterns: [ + { + include: '#comment', + }, + { + begin: '(?x)(?=((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(:))', + end: '(?=,|\\})', + patterns: [ + { + include: '#object-binding-element-propertyName', + }, + { + include: '#binding-element-const', + }, + ], + }, + { + include: '#object-binding-pattern-const', + }, + { + include: '#destructuring-variable-rest-const', + }, + { + include: '#variable-initializer', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'object-binding-element-propertyName': { + begin: '(?x)(?=((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(:))', + end: '(:)', + endCaptures: { + '0': { + name: 'punctuation.destructuring.tsx', + }, + }, + patterns: [ + { + include: '#string', + }, + { + include: '#array-literal', + }, + { + include: '#numeric-literal', + }, + { + name: 'variable.object.property.tsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)', + }, + ], + }, + 'binding-element': { + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#numeric-literal', + }, + { + include: '#regex', + }, + { + include: '#object-binding-pattern', + }, + { + include: '#array-binding-pattern', + }, + { + include: '#destructuring-variable-rest', + }, + { + include: '#variable-initializer', + }, + ], + }, + 'binding-element-const': { + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#numeric-literal', + }, + { + include: '#regex', + }, + { + include: '#object-binding-pattern-const', + }, + { + include: '#array-binding-pattern-const', + }, + { + include: '#destructuring-variable-rest-const', + }, + { + include: '#variable-initializer', + }, + ], + }, + 'destructuring-variable-rest': { + match: '(?:(\\.\\.\\.)\\s*)?([_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'meta.definition.variable.tsx variable.other.readwrite.tsx', + }, + }, + }, + 'destructuring-variable-rest-const': { + match: '(?:(\\.\\.\\.)\\s*)?([_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'meta.definition.variable.tsx variable.other.constant.tsx', + }, + }, + }, + 'object-binding-pattern': { + begin: '(?:(\\.\\.\\.)\\s*)?(\\{)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + patterns: [ + { + include: '#object-binding-element', + }, + ], + }, + 'object-binding-pattern-const': { + begin: '(?:(\\.\\.\\.)\\s*)?(\\{)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + patterns: [ + { + include: '#object-binding-element-const', + }, + ], + }, + 'array-binding-pattern': { + begin: '(?:(\\.\\.\\.)\\s*)?(\\[)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + patterns: [ + { + include: '#binding-element', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'array-binding-pattern-const': { + begin: '(?:(\\.\\.\\.)\\s*)?(\\[)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + patterns: [ + { + include: '#binding-element-const', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'parameter-name': { + patterns: [ + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(override|public|protected|private|readonly)\\s+(?=(override|public|protected|private|readonly)\\s+)', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + }, + }, + { + match: '(?x)(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(override|public|private|protected|readonly)\\s+)?(?:(\\.\\.\\.)\\s*)?(?<!=|:)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*(\\??)(?=\\s*\n# function assignment |\n(=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Function(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))) |\n(:\\s*((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.operator.rest.tsx', + }, + '3': { + name: 'entity.name.function.tsx variable.language.this.tsx', + }, + '4': { + name: 'entity.name.function.tsx', + }, + '5': { + name: 'keyword.operator.optional.tsx', + }, + }, + }, + { + match: '(?x)(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(override|public|private|protected|readonly)\\s+)?(?:(\\.\\.\\.)\\s*)?(?<!=|:)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*(\\??)', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.operator.rest.tsx', + }, + '3': { + name: 'variable.parameter.tsx variable.language.this.tsx', + }, + '4': { + name: 'variable.parameter.tsx', + }, + '5': { + name: 'keyword.operator.optional.tsx', + }, + }, + }, + ], + }, + 'destructuring-parameter': { + patterns: [ + { + name: 'meta.parameter.object-binding-pattern.tsx', + begin: '(?<!=|:)\\s*(?:(\\.\\.\\.)\\s*)?(\\{)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + patterns: [ + { + include: '#parameter-object-binding-element', + }, + ], + }, + { + name: 'meta.paramter.array-binding-pattern.tsx', + begin: '(?<!=|:)\\s*(?:(\\.\\.\\.)\\s*)?(\\[)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + patterns: [ + { + include: '#parameter-binding-element', + }, + { + include: '#punctuation-comma', + }, + ], + }, + ], + }, + 'parameter-object-binding-element': { + patterns: [ + { + include: '#comment', + }, + { + begin: '(?x)(?=((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(:))', + end: '(?=,|\\})', + patterns: [ + { + include: '#object-binding-element-propertyName', + }, + { + include: '#parameter-binding-element', + }, + { + include: '#paren-expression', + }, + ], + }, + { + include: '#parameter-object-binding-pattern', + }, + { + include: '#destructuring-parameter-rest', + }, + { + include: '#variable-initializer', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'parameter-binding-element': { + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#numeric-literal', + }, + { + include: '#regex', + }, + { + include: '#parameter-object-binding-pattern', + }, + { + include: '#parameter-array-binding-pattern', + }, + { + include: '#destructuring-parameter-rest', + }, + { + include: '#variable-initializer', + }, + ], + }, + 'destructuring-parameter-rest': { + match: '(?:(\\.\\.\\.)\\s*)?([_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'variable.parameter.tsx', + }, + }, + }, + 'parameter-object-binding-pattern': { + begin: '(?:(\\.\\.\\.)\\s*)?(\\{)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.object.tsx', + }, + }, + patterns: [ + { + include: '#parameter-object-binding-element', + }, + ], + }, + 'parameter-array-binding-pattern': { + begin: '(?:(\\.\\.\\.)\\s*)?(\\[)', + beginCaptures: { + '1': { + name: 'keyword.operator.rest.tsx', + }, + '2': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'punctuation.definition.binding-pattern.array.tsx', + }, + }, + patterns: [ + { + include: '#parameter-binding-element', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'field-declaration': { + name: 'meta.field.declaration.tsx', + begin: '(?x)(?<!\\()(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(readonly)\\s+)?(?=\\s*((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|(\\#?[_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(?:(?:(\\?)|(\\!))\\s*)?(=|:|;|,|\\}|$))', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + }, + end: '(?x)(?=\\}|;|,|$|(^(?!\\s*((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|(\\#?[_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(?:(?:(\\?)|(\\!))\\s*)?(=|:|;|,|$))))|(?<=\\})', + patterns: [ + { + include: '#variable-initializer', + }, + { + include: '#type-annotation', + }, + { + include: '#string', + }, + { + include: '#array-literal', + }, + { + include: '#numeric-literal', + }, + { + include: '#comment', + }, + { + match: '(?x)(\\#?[_$[:alpha:]][_$[:alnum:]]*)(?:(\\?)|(\\!))?(?=\\s*\\s*\n# function assignment |\n(=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Function(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))) |\n(:\\s*((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '1': { + name: 'meta.definition.property.tsx entity.name.function.tsx', + }, + '2': { + name: 'keyword.operator.optional.tsx', + }, + '3': { + name: 'keyword.operator.definiteassignment.tsx', + }, + }, + }, + { + name: 'meta.definition.property.tsx variable.object.property.tsx', + match: '\\#?[_$[:alpha:]][_$[:alnum:]]*', + }, + { + name: 'keyword.operator.optional.tsx', + match: '\\?', + }, + { + name: 'keyword.operator.definiteassignment.tsx', + match: '\\!', + }, + ], + }, + 'variable-initializer': { + patterns: [ + { + begin: '(?<!=|!)(=)(?!=)(?=\\s*\\S)(?!\\s*.*=>\\s*$)', + beginCaptures: { + '1': { + name: 'keyword.operator.assignment.tsx', + }, + }, + end: '(?=$|^|[,);}\\]]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + begin: '(?<!=|!)(=)(?!=)', + beginCaptures: { + '1': { + name: 'keyword.operator.assignment.tsx', + }, + }, + end: '(?=[,);}\\]]|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(of|in)\\s+))|(?=^\\s*$)|(?<![\\|\\&\\+\\-\\*\\/])(?<=\\S)(?<!=)(?=\\s*$)', + patterns: [ + { + include: '#expression', + }, + ], + }, + ], + }, + 'function-declaration': { + name: 'meta.function.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?(?:(async)\\s+)?(function\\b)(?:\\s*(\\*))?(?:(?:\\s+|(?<=\\*))([_$[:alpha:]][_$[:alnum:]]*))?\\s*', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.async.tsx', + }, + '4': { + name: 'storage.type.function.tsx', + }, + '5': { + name: 'keyword.generator.asterisk.tsx', + }, + '6': { + name: 'meta.definition.function.tsx entity.name.function.tsx', + }, + }, + end: '(?=;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))|(?<=\\})', + patterns: [ + { + include: '#function-name', + }, + { + include: '#function-body', + }, + ], + }, + 'function-expression': { + name: 'meta.function.expression.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(async)\\s+)?(function\\b)(?:\\s*(\\*))?(?:(?:\\s+|(?<=\\*))([_$[:alpha:]][_$[:alnum:]]*))?\\s*', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + '2': { + name: 'storage.type.function.tsx', + }, + '3': { + name: 'keyword.generator.asterisk.tsx', + }, + '4': { + name: 'meta.definition.function.tsx entity.name.function.tsx', + }, + }, + end: '(?=;)|(?<=\\})', + patterns: [ + { + include: '#function-name', + }, + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#function-body', + }, + ], + }, + 'function-name': { + name: 'meta.definition.function.tsx entity.name.function.tsx', + match: '[_$[:alpha:]][_$[:alnum:]]*', + }, + 'function-body': { + patterns: [ + { + include: '#comment', + }, + { + include: '#type-parameters', + }, + { + include: '#function-parameters', + }, + { + include: '#return-type', + }, + { + include: '#type-function-return-type', + }, + { + include: '#decl-block', + }, + { + name: 'keyword.generator.asterisk.tsx', + match: '\\*', + }, + ], + }, + 'method-declaration': { + patterns: [ + { + name: 'meta.method.declaration.tsx', + begin: '(?x)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:\\b(override)\\s+)?(?:\\b(public|private|protected)\\s+)?(?:\\b(abstract)\\s+)?(?:\\b(async)\\s+)?\\s*\\b(constructor)\\b(?!:)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.tsx', + }, + '4': { + name: 'storage.modifier.async.tsx', + }, + '5': { + name: 'storage.type.tsx', + }, + }, + end: '(?=\\}|;|,|$)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + ], + }, + { + name: 'meta.method.declaration.tsx', + begin: '(?x)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:\\b(override)\\s+)?(?:\\b(public|private|protected)\\s+)?(?:\\b(abstract)\\s+)?(?:\\b(async)\\s+)?(?:(?:\\s*\\b(new)\\b(?!:)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))|(?:(\\*)\\s*)?)(?=\\s*((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.tsx', + }, + '4': { + name: 'storage.modifier.async.tsx', + }, + '5': { + name: 'keyword.operator.new.tsx', + }, + '6': { + name: 'keyword.generator.asterisk.tsx', + }, + }, + end: '(?=\\}|;|,|$)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + ], + }, + { + name: 'meta.method.declaration.tsx', + begin: '(?x)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:\\b(override)\\s+)?(?:\\b(public|private|protected)\\s+)?(?:\\b(abstract)\\s+)?(?:\\b(async)\\s+)?(?:\\b(get|set)\\s+)?(?:(\\*)\\s*)?(?=\\s*(((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(\\??))\\s*((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.tsx', + }, + '4': { + name: 'storage.modifier.async.tsx', + }, + '5': { + name: 'storage.type.property.tsx', + }, + '6': { + name: 'keyword.generator.asterisk.tsx', + }, + }, + end: '(?=\\}|;|,|$)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + ], + }, + ], + }, + 'object-literal-method-declaration': { + name: 'meta.method.declaration.tsx', + begin: '(?x)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:\\b(async)\\s+)?(?:\\b(get|set)\\s+)?(?:(\\*)\\s*)?(?=\\s*(((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(\\??))\\s*((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + '2': { + name: 'storage.type.property.tsx', + }, + '3': { + name: 'keyword.generator.asterisk.tsx', + }, + }, + end: '(?=\\}|;|,)|(?<=\\})', + patterns: [ + { + include: '#method-declaration-name', + }, + { + include: '#function-body', + }, + { + begin: '(?x)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:\\b(async)\\s+)?(?:\\b(get|set)\\s+)?(?:(\\*)\\s*)?(?=\\s*(((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(\\??))\\s*((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?[\\(])', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + '2': { + name: 'storage.type.property.tsx', + }, + '3': { + name: 'keyword.generator.asterisk.tsx', + }, + }, + end: '(?=\\(|\\<)', + patterns: [ + { + include: '#method-declaration-name', + }, + ], + }, + ], + }, + 'method-declaration-name': { + begin: '(?x)(?=((\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$))|([_$[:alpha:]][_$[:alnum:]]*)|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\]))\\s*(\\??)\\s*[\\(\\<])', + end: '(?=\\(|\\<)', + patterns: [ + { + include: '#string', + }, + { + include: '#array-literal', + }, + { + include: '#numeric-literal', + }, + { + name: 'meta.definition.method.tsx entity.name.function.tsx', + match: '[_$[:alpha:]][_$[:alnum:]]*', + }, + { + name: 'keyword.operator.optional.tsx', + match: '\\?', + }, + ], + }, + 'arrow-function': { + patterns: [ + { + name: 'meta.arrow.tsx', + match: '(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(\\basync)\\s+)?([_$[:alpha:]][_$[:alnum:]]*)\\s*(?==>)', + captures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + '2': { + name: 'variable.parameter.tsx', + }, + }, + }, + { + name: 'meta.arrow.tsx', + begin: '(?x) (?:\n (?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(\\basync)\n)? ((?<![})!\\]])\\s*\n (?=\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n )\n)', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + }, + end: '(?==>|\\{|(^\\s*(export|function|class|interface|let|var|const|import|enum|namespace|module|type|abstract|declare)\\s+))', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-parameters', + }, + { + include: '#function-parameters', + }, + { + include: '#arrow-return-type', + }, + { + include: '#possibly-arrow-return-type', + }, + ], + }, + { + name: 'meta.arrow.tsx', + begin: '=>', + beginCaptures: { + '0': { + name: 'storage.type.function.arrow.tsx', + }, + }, + end: '((?<=\\}|\\S)(?<!=>)|((?!\\{)(?=\\S)))(?!\\/[\\/\\*])', + patterns: [ + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#decl-block', + }, + { + include: '#expression', + }, + ], + }, + ], + }, + 'indexer-declaration': { + name: 'meta.indexer.declaration.tsx', + begin: '(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(readonly)\\s*)?\\s*(\\[)\\s*([_$[:alpha:]][_$[:alnum:]]*)\\s*(?=:)', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'meta.brace.square.tsx', + }, + '3': { + name: 'variable.parameter.tsx', + }, + }, + end: '(\\])\\s*(\\?\\s*)?|$', + endCaptures: { + '1': { + name: 'meta.brace.square.tsx', + }, + '2': { + name: 'keyword.operator.optional.tsx', + }, + }, + patterns: [ + { + include: '#type-annotation', + }, + ], + }, + 'indexer-mapped-type-declaration': { + name: 'meta.indexer.mappedtype.declaration.tsx', + begin: '(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))([+-])?(readonly)\\s*)?\\s*(\\[)\\s*([_$[:alpha:]][_$[:alnum:]]*)\\s+(in)\\s+', + beginCaptures: { + '1': { + name: 'keyword.operator.type.modifier.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'meta.brace.square.tsx', + }, + '4': { + name: 'entity.name.type.tsx', + }, + '5': { + name: 'keyword.operator.expression.in.tsx', + }, + }, + end: '(\\])([+-])?\\s*(\\?\\s*)?|$', + endCaptures: { + '1': { + name: 'meta.brace.square.tsx', + }, + '2': { + name: 'keyword.operator.type.modifier.tsx', + }, + '3': { + name: 'keyword.operator.optional.tsx', + }, + }, + patterns: [ + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(as)\\s+', + captures: { + '1': { + name: 'keyword.control.as.tsx', + }, + }, + }, + { + include: '#type', + }, + ], + }, + 'function-parameters': { + name: 'meta.parameters.tsx', + begin: '\\(', + beginCaptures: { + '0': { + name: 'punctuation.definition.parameters.begin.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'punctuation.definition.parameters.end.tsx', + }, + }, + patterns: [ + { + include: '#function-parameters-body', + }, + ], + }, + 'function-parameters-body': { + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#decorator', + }, + { + include: '#destructuring-parameter', + }, + { + include: '#parameter-name', + }, + { + include: '#parameter-type-annotation', + }, + { + include: '#variable-initializer', + }, + { + name: 'punctuation.separator.parameter.tsx', + match: ',', + }, + ], + }, + 'class-declaration': { + name: 'meta.class.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(?:(abstract)\\s+)?\\b(class)\\b(?=\\s+|/[/*])', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.tsx', + }, + '4': { + name: 'storage.type.class.tsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#class-declaration-or-expression-patterns', + }, + ], + }, + 'class-expression': { + name: 'meta.class.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(abstract)\\s+)?(class)\\b(?=\\s+|[<{]|\\/[\\/*])', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'storage.type.class.tsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#class-declaration-or-expression-patterns', + }, + ], + }, + 'class-declaration-or-expression-patterns': { + patterns: [ + { + include: '#comment', + }, + { + include: '#class-or-interface-heritage', + }, + { + match: '[_$[:alpha:]][_$[:alnum:]]*', + captures: { + '0': { + name: 'entity.name.type.class.tsx', + }, + }, + }, + { + include: '#type-parameters', + }, + { + include: '#class-or-interface-body', + }, + ], + }, + 'interface-declaration': { + name: 'meta.interface.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(?:(abstract)\\s+)?\\b(interface)\\b(?=\\s+|/[/*])', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.tsx', + }, + '4': { + name: 'storage.type.interface.tsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#comment', + }, + { + include: '#class-or-interface-heritage', + }, + { + match: '[_$[:alpha:]][_$[:alnum:]]*', + captures: { + '0': { + name: 'entity.name.type.interface.tsx', + }, + }, + }, + { + include: '#type-parameters', + }, + { + include: '#class-or-interface-body', + }, + ], + }, + 'class-or-interface-heritage': { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:\\b(extends|implements)\\b)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + }, + end: '(?=\\{)', + patterns: [ + { + include: '#comment', + }, + { + include: '#class-or-interface-heritage', + }, + { + include: '#type-parameters', + }, + { + include: '#expressionWithoutIdentifiers', + }, + { + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))(?=\\s*[_$[:alpha:]][_$[:alnum:]]*(\\s*\\??\\.\\s*[_$[:alpha:]][_$[:alnum:]]*)*\\s*)', + captures: { + '1': { + name: 'entity.name.type.module.tsx', + }, + '2': { + name: 'punctuation.accessor.tsx', + }, + '3': { + name: 'punctuation.accessor.optional.tsx', + }, + }, + }, + { + match: '([_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'entity.other.inherited-class.tsx', + }, + }, + }, + { + include: '#expressionPunctuations', + }, + ], + }, + 'class-or-interface-body': { + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + include: '#decorator', + }, + { + begin: '(?<=:)\\s*', + end: '(?=\\s|[;),}\\]:\\-\\+]|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + include: '#method-declaration', + }, + { + include: '#indexer-declaration', + }, + { + include: '#field-declaration', + }, + { + include: '#string', + }, + { + include: '#type-annotation', + }, + { + include: '#variable-initializer', + }, + { + include: '#access-modifier', + }, + { + include: '#property-accessor', + }, + { + include: '#async-modifier', + }, + { + include: '#after-operator-block-as-object-literal', + }, + { + include: '#decl-block', + }, + { + include: '#expression', + }, + { + include: '#punctuation-comma', + }, + { + include: '#punctuation-semicolon', + }, + ], + }, + 'access-modifier': { + name: 'storage.modifier.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(abstract|declare|override|public|protected|private|readonly|static)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'property-accessor': { + name: 'storage.type.property.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(accessor|get|set)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'async-modifier': { + name: 'storage.modifier.async.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(async)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'enum-declaration': { + name: 'meta.enum.declaration.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?(?:\\b(const)\\s+)?\\b(enum)\\s+([_$[:alpha:]][_$[:alnum:]]*)', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.modifier.tsx', + }, + '4': { + name: 'storage.type.enum.tsx', + }, + '5': { + name: 'entity.name.type.enum.tsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#comment', + }, + { + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)', + beginCaptures: { + '0': { + name: 'variable.other.enummember.tsx', + }, + }, + end: '(?=,|\\}|$)', + patterns: [ + { + include: '#comment', + }, + { + include: '#variable-initializer', + }, + ], + }, + { + begin: '(?=((\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])+\\])))', + end: '(?=,|\\}|$)', + patterns: [ + { + include: '#string', + }, + { + include: '#array-literal', + }, + { + include: '#comment', + }, + { + include: '#variable-initializer', + }, + ], + }, + { + include: '#punctuation-comma', + }, + ], + }, + ], + }, + 'namespace-declaration': { + name: 'meta.namespace.declaration.tsx', + begin: '(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(namespace|module)\\s+(?=[_$[:alpha:]"\'`]))', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.type.namespace.tsx', + }, + }, + end: '(?<=\\})|(?=;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + name: 'entity.name.type.module.tsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)', + }, + { + include: '#punctuation-accessor', + }, + { + include: '#decl-block', + }, + ], + }, + 'type-alias-declaration': { + name: 'meta.type.declaration.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(type)\\b\\s+([_$[:alpha:]][_$[:alnum:]]*)\\s*', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'storage.type.type.tsx', + }, + '4': { + name: 'entity.name.type.alias.tsx', + }, + }, + end: '(?=\\}|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-parameters', + }, + { + begin: '(=)\\s*(intrinsic)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'keyword.operator.assignment.tsx', + }, + '2': { + name: 'keyword.control.intrinsic.tsx', + }, + }, + end: '(?=\\}|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#type', + }, + ], + }, + { + begin: '(=)\\s*', + beginCaptures: { + '1': { + name: 'keyword.operator.assignment.tsx', + }, + }, + end: '(?=\\}|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#type', + }, + ], + }, + ], + }, + 'import-equals-declaration': { + patterns: [ + { + name: 'meta.import-equals.external.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(import)(?:\\s+(type))?\\s+([_$[:alpha:]][_$[:alnum:]]*)\\s*(=)\\s*(require)\\s*(\\()', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'keyword.control.import.tsx', + }, + '4': { + name: 'keyword.control.type.tsx', + }, + '5': { + name: 'variable.other.readwrite.alias.tsx', + }, + '6': { + name: 'keyword.operator.assignment.tsx', + }, + '7': { + name: 'keyword.control.require.tsx', + }, + '8': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + ], + }, + { + name: 'meta.import-equals.internal.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(import)(?:\\s+(type))?\\s+([_$[:alpha:]][_$[:alnum:]]*)\\s*(=)\\s*(?!require\\b)', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'keyword.control.import.tsx', + }, + '4': { + name: 'keyword.control.type.tsx', + }, + '5': { + name: 'variable.other.readwrite.alias.tsx', + }, + '6': { + name: 'keyword.operator.assignment.tsx', + }, + }, + end: '(?=;|$|^)', + patterns: [ + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#comment', + }, + { + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))', + captures: { + '1': { + name: 'entity.name.type.module.tsx', + }, + '2': { + name: 'punctuation.accessor.tsx', + }, + '3': { + name: 'punctuation.accessor.optional.tsx', + }, + }, + }, + { + name: 'variable.other.readwrite.tsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)', + }, + ], + }, + ], + }, + 'import-declaration': { + name: 'meta.import.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(\\bexport)\\s+)?(?:(\\bdeclare)\\s+)?\\b(import)(?:\\s+(type)(?!\\s+from))?(?!\\s*[:\\(])(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + '3': { + name: 'keyword.control.import.tsx', + }, + '4': { + name: 'keyword.control.type.tsx', + }, + }, + end: '(?<!^import|[^\\._$[:alnum:]]import)(?=;|$|^)', + patterns: [ + { + include: '#single-line-comment-consuming-line-ending', + }, + { + include: '#comment', + }, + { + include: '#string', + }, + { + begin: '(?<=^import|[^\\._$[:alnum:]]import)(?!\\s*["\'])', + end: '\\bfrom\\b', + endCaptures: { + '0': { + name: 'keyword.control.from.tsx', + }, + }, + patterns: [ + { + include: '#import-export-declaration', + }, + ], + }, + { + include: '#import-export-declaration', + }, + ], + }, + 'export-declaration': { + patterns: [ + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(export)\\s+(as)\\s+(namespace)\\s+([_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'keyword.control.as.tsx', + }, + '3': { + name: 'storage.type.namespace.tsx', + }, + '4': { + name: 'entity.name.type.module.tsx', + }, + }, + }, + { + name: 'meta.export.default.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(export)(?:\\s+(type))?(?:(?:\\s*(=))|(?:\\s+(default)(?=\\s+)))', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'keyword.control.type.tsx', + }, + '3': { + name: 'keyword.operator.assignment.tsx', + }, + '4': { + name: 'keyword.control.default.tsx', + }, + }, + end: '(?=$|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#interface-declaration', + }, + { + include: '#expression', + }, + ], + }, + { + name: 'meta.export.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(export)(?:\\s+(type))?\\b(?!(\\$)|(\\s*:))((?=\\s*[\\{*])|((?=\\s*[_$[:alpha:]][_$[:alnum:]]*(\\s|,))(?!\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b)))', + beginCaptures: { + '1': { + name: 'keyword.control.export.tsx', + }, + '2': { + name: 'keyword.control.type.tsx', + }, + }, + end: '(?=$|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#import-export-declaration', + }, + ], + }, + ], + }, + 'import-export-declaration': { + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#import-export-block', + }, + { + name: 'keyword.control.from.tsx', + match: '\\bfrom\\b', + }, + { + include: '#import-export-assert-clause', + }, + { + include: '#import-export-clause', + }, + ], + }, + 'import-export-assert-clause': { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(assert)\\s*(\\{)', + beginCaptures: { + '1': { + name: 'keyword.control.assert.tsx', + }, + '2': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + { + name: 'meta.object-literal.key.tsx', + match: '(?:[_$[:alpha:]][_$[:alnum:]]*)\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*:)', + }, + { + name: 'punctuation.separator.key-value.tsx', + match: ':', + }, + ], + }, + 'import-export-block': { + name: 'meta.block.tsx', + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#import-export-clause', + }, + ], + }, + 'import-export-clause': { + patterns: [ + { + include: '#comment', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(?:(\\btype)\\s+)?(?:(\\bdefault)|(\\*)|(\\b[_$[:alpha:]][_$[:alnum:]]*)))\\s+(as)\\s+(?:(default(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))|([_$[:alpha:]][_$[:alnum:]]*))', + captures: { + '1': { + name: 'keyword.control.type.tsx', + }, + '2': { + name: 'keyword.control.default.tsx', + }, + '3': { + name: 'constant.language.import-export-all.tsx', + }, + '4': { + name: 'variable.other.readwrite.tsx', + }, + '5': { + name: 'keyword.control.as.tsx', + }, + '6': { + name: 'keyword.control.default.tsx', + }, + '7': { + name: 'variable.other.readwrite.alias.tsx', + }, + }, + }, + { + include: '#punctuation-comma', + }, + { + name: 'constant.language.import-export-all.tsx', + match: '\\*', + }, + { + name: 'keyword.control.default.tsx', + match: '\\b(default)\\b', + }, + { + match: '(?:(\\btype)\\s+)?([_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'keyword.control.type.tsx', + }, + '2': { + name: 'variable.other.readwrite.alias.tsx', + }, + }, + }, + ], + }, + 'switch-statement': { + name: 'switch-statement.expr.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?=\\bswitch\\s*\\()', + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + name: 'switch-expression.expr.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(switch)\\s*(\\()', + beginCaptures: { + '1': { + name: 'keyword.control.switch.tsx', + }, + '2': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + }, + { + name: 'switch-block.expr.tsx', + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '(?=\\})', + patterns: [ + { + name: 'case-clause.expr.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(case|default(?=:))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'keyword.control.switch.tsx', + }, + }, + end: '(?=:)', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + begin: '(:)\\s*(\\{)', + beginCaptures: { + '1': { + name: 'case-clause.expr.tsx punctuation.definition.section.case-statement.tsx', + }, + '2': { + name: 'meta.block.tsx punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'meta.block.tsx punctuation.definition.block.tsx', + }, + }, + contentName: 'meta.block.tsx', + patterns: [ + { + include: '#statements', + }, + ], + }, + { + match: '(:)', + captures: { + '0': { + name: 'case-clause.expr.tsx punctuation.definition.section.case-statement.tsx', + }, + }, + }, + { + include: '#statements', + }, + ], + }, + ], + }, + 'for-loop': { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))for(?=((\\s+|(\\s*\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*))await)?\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)?(\\())', + beginCaptures: { + '0': { + name: 'keyword.control.loop.tsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#comment', + }, + { + name: 'keyword.control.loop.tsx', + match: 'await', + }, + { + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#var-expr', + }, + { + include: '#expression', + }, + { + include: '#punctuation-semicolon', + }, + ], + }, + ], + }, + 'if-statement': { + patterns: [ + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?=\\bif\\s*(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))\\s*(?!\\{))', + end: '(?=;|$|\\})', + patterns: [ + { + include: '#comment', + }, + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(if)\\s*(\\()', + beginCaptures: { + '1': { + name: 'keyword.control.conditional.tsx', + }, + '2': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + }, + { + name: 'string.regexp.tsx', + begin: '(?<=\\))\\s*\\/(?![\\/*])(?=(?:[^\\/\\\\\\[]|\\\\.|\\[([^\\]\\\\]|\\\\.)*\\])+\\/([dgimsuy]+|(?![\\/\\*])|(?=\\/\\*))(?!\\s*[a-zA-Z0-9_$]))', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + end: '(/)([dgimsuy]*)', + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.tsx', + }, + '2': { + name: 'keyword.other.tsx', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + { + include: '#statements', + }, + ], + }, + ], + }, + 'decl-block': { + name: 'meta.block.tsx', + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#statements', + }, + ], + }, + 'after-operator-block-as-object-literal': { + name: 'meta.objectliteral.tsx', + begin: '(?<!\\+\\+|--)(?<=[:=(,\\[?+!>]|^await|[^\\._$[:alnum:]]await|^return|[^\\._$[:alnum:]]return|^yield|[^\\._$[:alnum:]]yield|^throw|[^\\._$[:alnum:]]throw|^in|[^\\._$[:alnum:]]in|^of|[^\\._$[:alnum:]]of|^typeof|[^\\._$[:alnum:]]typeof|&&|\\|\\||\\*)\\s*(\\{)', + beginCaptures: { + '1': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#object-member', + }, + ], + }, + 'object-literal': { + name: 'meta.objectliteral.tsx', + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#object-member', + }, + ], + }, + 'object-member': { + patterns: [ + { + include: '#comment', + }, + { + include: '#object-literal-method-declaration', + }, + { + name: 'meta.object.member.tsx meta.object-literal.key.tsx', + begin: '(?=\\[)', + end: '(?=:)|((?<=[\\]])(?=\\s*[\\(\\<]))', + patterns: [ + { + include: '#comment', + }, + { + include: '#array-literal', + }, + ], + }, + { + name: 'meta.object.member.tsx meta.object-literal.key.tsx', + begin: '(?=[\\\'\\"\\`])', + end: '(?=:)|((?<=[\\\'\\"\\`])(?=((\\s*[\\(\\<,}])|(\\s+(as|satisifies)\\s+))))', + patterns: [ + { + include: '#comment', + }, + { + include: '#string', + }, + ], + }, + { + name: 'meta.object.member.tsx meta.object-literal.key.tsx', + begin: '(?x)(?=(\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$))|(\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$))|((?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$)))', + end: '(?=:)|(?=\\s*([\\(\\<,}])|(\\s+as|satisifies\\s+))', + patterns: [ + { + include: '#comment', + }, + { + include: '#numeric-literal', + }, + ], + }, + { + name: 'meta.method.declaration.tsx', + begin: '(?<=[\\]\\\'\\"\\`])(?=\\s*[\\(\\<])', + end: '(?=\\}|;|,)|(?<=\\})', + patterns: [ + { + include: '#function-body', + }, + ], + }, + { + name: 'meta.object.member.tsx', + match: '(?![_$[:alpha:]])([[:digit:]]+)\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*:)', + captures: { + '0': { + name: 'meta.object-literal.key.tsx', + }, + '1': { + name: 'constant.numeric.decimal.tsx', + }, + }, + }, + { + name: 'meta.object.member.tsx', + match: '(?x)(?:([_$[:alpha:]][_$[:alnum:]]*)\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*:(\\s*\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/)*\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '0': { + name: 'meta.object-literal.key.tsx', + }, + '1': { + name: 'entity.name.function.tsx', + }, + }, + }, + { + name: 'meta.object.member.tsx', + match: '(?:[_$[:alpha:]][_$[:alnum:]]*)\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*:)', + captures: { + '0': { + name: 'meta.object-literal.key.tsx', + }, + }, + }, + { + name: 'meta.object.member.tsx', + begin: '\\.\\.\\.', + beginCaptures: { + '0': { + name: 'keyword.operator.spread.tsx', + }, + }, + end: '(?=,|\\})', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + name: 'meta.object.member.tsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?=,|\\}|$|\\/\\/|\\/\\*)', + captures: { + '1': { + name: 'variable.other.readwrite.tsx', + }, + }, + }, + { + name: 'meta.object.member.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(as)\\s+(const)(?=\\s*([,}]|$))', + captures: { + '1': { + name: 'keyword.control.as.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + }, + }, + { + name: 'meta.object.member.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(as)|(satisfies))\\s+', + beginCaptures: { + '1': { + name: 'keyword.control.as.tsx', + }, + '2': { + name: 'keyword.control.satisfies.tsx', + }, + }, + end: '(?=[;),}\\]:?\\-\\+\\>]|\\|\\||\\&\\&|\\!\\=\\=|$|^|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(as|satisifies)\\s+))', + patterns: [ + { + include: '#type', + }, + ], + }, + { + name: 'meta.object.member.tsx', + begin: '(?=[_$[:alpha:]][_$[:alnum:]]*\\s*=)', + end: '(?=,|\\}|$|\\/\\/|\\/\\*)', + patterns: [ + { + include: '#expression', + }, + ], + }, + { + name: 'meta.object.member.tsx', + begin: ':', + beginCaptures: { + '0': { + name: 'meta.object-literal.key.tsx punctuation.separator.key-value.tsx', + }, + }, + end: '(?=,|\\})', + patterns: [ + { + begin: '(?<=:)\\s*(async)?(?=\\s*(<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)\\(\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#type-parameters', + }, + { + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + ], + }, + { + begin: '(?<=:)\\s*(async)?\\s*(\\()(?=\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + '2': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + { + begin: '(?<=:)\\s*(async)?\\s*(?=\\<\\s*$)', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + }, + end: '(?<=\\>)', + patterns: [ + { + include: '#type-parameters', + }, + ], + }, + { + begin: '(?<=\\>)\\s*(\\()(?=\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + { + include: '#possibly-arrow-return-type', + }, + { + include: '#expression', + }, + ], + }, + { + include: '#punctuation-comma', + }, + { + include: '#decl-block', + }, + ], + }, + 'ternary-expression': { + begin: '(?!\\?\\.\\s*[^[:digit:]])(\\?)(?!\\?)', + beginCaptures: { + '1': { + name: 'keyword.operator.ternary.tsx', + }, + }, + end: '\\s*(:)', + endCaptures: { + '1': { + name: 'keyword.operator.ternary.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + }, + 'function-call': { + patterns: [ + { + begin: '(?=(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))\\s*(?:(\\?\\.\\s*)|(\\!))?((<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?<!=)\\>))*(?<!=)\\>)*(?<!=)>\\s*)?\\())', + end: '(?<=\\))(?!(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))\\s*(?:(\\?\\.\\s*)|(\\!))?((<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?<!=)\\>))*(?<!=)\\>)*(?<!=)>\\s*)?\\())', + patterns: [ + { + name: 'meta.function-call.tsx', + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))', + end: '(?=\\s*(?:(\\?\\.\\s*)|(\\!))?((<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?<!=)\\>))*(?<!=)\\>)*(?<!=)>\\s*)?\\())', + patterns: [ + { + include: '#function-call-target', + }, + ], + }, + { + include: '#comment', + }, + { + include: '#function-call-optionals', + }, + { + include: '#type-arguments', + }, + { + include: '#paren-expression', + }, + ], + }, + { + begin: '(?=(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))(<\\s*[\\{\\[\\(]\\s*$))', + end: '(?<=\\>)(?!(((([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))|(?<=[\\)]))(<\\s*[\\{\\[\\(]\\s*$))', + patterns: [ + { + name: 'meta.function-call.tsx', + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*)(\\s*\\??\\.\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*))*)|(\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*))', + end: '(?=(<\\s*[\\{\\[\\(]\\s*$))', + patterns: [ + { + include: '#function-call-target', + }, + ], + }, + { + include: '#comment', + }, + { + include: '#function-call-optionals', + }, + { + include: '#type-arguments', + }, + ], + }, + ], + }, + 'function-call-target': { + patterns: [ + { + include: '#support-function-call-identifiers', + }, + { + name: 'entity.name.function.tsx', + match: '(\\#?[_$[:alpha:]][_$[:alnum:]]*)', + }, + ], + }, + 'function-call-optionals': { + patterns: [ + { + name: 'meta.function-call.tsx punctuation.accessor.optional.tsx', + match: '\\?\\.', + }, + { + name: 'meta.function-call.tsx keyword.operator.definiteassignment.tsx', + match: '\\!', + }, + ], + }, + 'support-function-call-identifiers': { + patterns: [ + { + include: '#literal', + }, + { + include: '#support-objects', + }, + { + include: '#object-identifiers', + }, + { + include: '#punctuation-accessor', + }, + { + name: 'keyword.operator.expression.import.tsx', + match: '(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))import(?=\\s*[\\(]\\s*[\\"\\\'\\`]))', + }, + ], + }, + 'new-expr': { + name: 'new.expr.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(new)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'keyword.operator.new.tsx', + }, + }, + end: '(?<=\\))|(?=[;),}\\]:?\\-\\+\\>]|\\|\\||\\&\\&|\\!\\=\\=|$|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))new(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))function((\\s+[_$[:alpha:]][_$[:alnum:]]*)|(\\s*[\\(]))))', + patterns: [ + { + include: '#expression', + }, + ], + }, + 'instanceof-expr': { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(instanceof)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '1': { + name: 'keyword.operator.expression.instanceof.tsx', + }, + }, + end: '(?<=\\))|(?=[;),}\\]:?\\-\\+\\>]|\\|\\||\\&\\&|\\!\\=\\=|$|(===|!==|==|!=)|(([\\&\\~\\^\\|]\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s+instanceof(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))function((\\s+[_$[:alpha:]][_$[:alnum:]]*)|(\\s*[\\(]))))', + patterns: [ + { + include: '#type', + }, + ], + }, + 'paren-expression-possibly-arrow': { + patterns: [ + { + begin: '(?<=[(=,])\\s*(async)?(?=\\s*((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?\\(\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#paren-expression-possibly-arrow-with-typeparameters', + }, + ], + }, + { + begin: '(?<=[(=,]|=>|^return|[^\\._$[:alnum:]]return)\\s*(async)?(?=\\s*((((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*))?\\()|(<)|((<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)))\\s*$)', + beginCaptures: { + '1': { + name: 'storage.modifier.async.tsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#paren-expression-possibly-arrow-with-typeparameters', + }, + ], + }, + { + include: '#possibly-arrow-return-type', + }, + ], + }, + 'paren-expression-possibly-arrow-with-typeparameters': { + patterns: [ + { + include: '#type-parameters', + }, + { + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression-inside-possibly-arrow-parens', + }, + ], + }, + ], + }, + 'expression-inside-possibly-arrow-parens': { + patterns: [ + { + include: '#expressionWithoutIdentifiers', + }, + { + include: '#comment', + }, + { + include: '#string', + }, + { + include: '#decorator', + }, + { + include: '#destructuring-parameter', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(override|public|protected|private|readonly)\\s+(?=(override|public|protected|private|readonly)\\s+)', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + }, + }, + { + match: '(?x)(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(override|public|private|protected|readonly)\\s+)?(?:(\\.\\.\\.)\\s*)?(?<!=|:)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*(\\??)(?=\\s*\n# function assignment |\n(=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)) |\n# typeannotation is fn type: < | () | (... | (param: | (param, | (param? | (param= | (param) =>\n(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Function(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))) |\n(:\\s*((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))) |\n(:\\s*(=>|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(<[^<>]*>)|[^<>(),=])+=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n)))', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.operator.rest.tsx', + }, + '3': { + name: 'entity.name.function.tsx variable.language.this.tsx', + }, + '4': { + name: 'entity.name.function.tsx', + }, + '5': { + name: 'keyword.operator.optional.tsx', + }, + }, + }, + { + match: '(?x)(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(override|public|private|protected|readonly)\\s+)?(?:(\\.\\.\\.)\\s*)?(?<!=|:)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*(\\??)(?=\\s*[:,]|$)', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.operator.rest.tsx', + }, + '3': { + name: 'variable.parameter.tsx variable.language.this.tsx', + }, + '4': { + name: 'variable.parameter.tsx', + }, + '5': { + name: 'keyword.operator.optional.tsx', + }, + }, + }, + { + include: '#type-annotation', + }, + { + include: '#variable-initializer', + }, + { + name: 'punctuation.separator.parameter.tsx', + match: ',', + }, + { + include: '#identifiers', + }, + { + include: '#expressionPunctuations', + }, + ], + }, + 'paren-expression': { + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + }, + cast: { + patterns: [ + { + include: '#jsx', + }, + ], + }, + 'expression-operators': { + patterns: [ + { + name: 'keyword.control.flow.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(await)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(yield)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))(?=\\s*\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*\\*)', + beginCaptures: { + '1': { + name: 'keyword.control.flow.tsx', + }, + }, + end: '\\*', + endCaptures: { + '0': { + name: 'keyword.generator.asterisk.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + ], + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(yield)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))(?:\\s*(\\*))?', + captures: { + '1': { + name: 'keyword.control.flow.tsx', + }, + '2': { + name: 'keyword.generator.asterisk.tsx', + }, + }, + }, + { + name: 'keyword.operator.expression.delete.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))delete(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.operator.expression.in.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))in(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))(?!\\()', + }, + { + name: 'keyword.operator.expression.of.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))of(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))(?!\\()', + }, + { + name: 'keyword.operator.expression.instanceof.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))instanceof(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.operator.new.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))new(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + include: '#typeof-operator', + }, + { + name: 'keyword.operator.expression.void.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))void(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(as)\\s+(const)(?=\\s*($|[;,:})\\]]))', + captures: { + '1': { + name: 'keyword.control.as.tsx', + }, + '2': { + name: 'storage.modifier.tsx', + }, + }, + }, + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(as)|(satisfies))\\s+', + beginCaptures: { + '1': { + name: 'keyword.control.as.tsx', + }, + '2': { + name: 'keyword.control.satisfies.tsx', + }, + }, + end: '(?=^|[;),}\\]:?\\-\\+\\>]|\\|\\||\\&\\&|\\!\\=\\=|$|((?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(as|satisfies)\\s+)|(\\s+\\<))', + patterns: [ + { + include: '#type', + }, + ], + }, + { + name: 'keyword.operator.spread.tsx', + match: '\\.\\.\\.', + }, + { + name: 'keyword.operator.assignment.compound.tsx', + match: '\\*=|(?<!\\()/=|%=|\\+=|\\-=', + }, + { + name: 'keyword.operator.assignment.compound.bitwise.tsx', + match: '\\&=|\\^=|<<=|>>=|>>>=|\\|=', + }, + { + name: 'keyword.operator.bitwise.shift.tsx', + match: '<<|>>>|>>', + }, + { + name: 'keyword.operator.comparison.tsx', + match: '===|!==|==|!=', + }, + { + name: 'keyword.operator.relational.tsx', + match: '<=|>=|<>|<|>', + }, + { + match: '(?<=[_$[:alnum:]])(\\!)\\s*(?:(/=)|(?:(/)(?![/*])))', + captures: { + '1': { + name: 'keyword.operator.logical.tsx', + }, + '2': { + name: 'keyword.operator.assignment.compound.tsx', + }, + '3': { + name: 'keyword.operator.arithmetic.tsx', + }, + }, + }, + { + name: 'keyword.operator.logical.tsx', + match: '\\!|&&|\\|\\||\\?\\?', + }, + { + name: 'keyword.operator.bitwise.tsx', + match: '\\&|~|\\^|\\|', + }, + { + name: 'keyword.operator.assignment.tsx', + match: '\\=', + }, + { + name: 'keyword.operator.decrement.tsx', + match: '--', + }, + { + name: 'keyword.operator.increment.tsx', + match: '\\+\\+', + }, + { + name: 'keyword.operator.arithmetic.tsx', + match: '%|\\*|/|-|\\+', + }, + { + begin: '(?<=[_$[:alnum:])\\]])\\s*(?=(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)+(?:(/=)|(?:(/)(?![/*]))))', + end: '(?:(/=)|(?:(/)(?!\\*([^\\*]|(\\*[^\\/]))*\\*\\/)))', + endCaptures: { + '1': { + name: 'keyword.operator.assignment.compound.tsx', + }, + '2': { + name: 'keyword.operator.arithmetic.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + ], + }, + { + match: '(?<=[_$[:alnum:])\\]])\\s*(?:(/=)|(?:(/)(?![/*])))', + captures: { + '1': { + name: 'keyword.operator.assignment.compound.tsx', + }, + '2': { + name: 'keyword.operator.arithmetic.tsx', + }, + }, + }, + ], + }, + 'typeof-operator': { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))typeof(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + beginCaptures: { + '0': { + name: 'keyword.operator.expression.typeof.tsx', + }, + }, + end: '(?=[,);}\\]=>:&|{\\?]|(extends\\s+)|$|;|^\\s*$|(?:^\\s*(?:abstract|async|class|const|declare|enum|export|function|import|interface|let|module|namespace|return|type|var)\\b))', + patterns: [ + { + include: '#type-arguments', + }, + { + include: '#expression', + }, + ], + }, + literal: { + patterns: [ + { + include: '#numeric-literal', + }, + { + include: '#boolean-literal', + }, + { + include: '#null-literal', + }, + { + include: '#undefined-literal', + }, + { + include: '#numericConstant-literal', + }, + { + include: '#array-literal', + }, + { + include: '#this-literal', + }, + { + include: '#super-literal', + }, + ], + }, + 'array-literal': { + name: 'meta.array.literal.tsx', + begin: '\\s*(\\[)', + beginCaptures: { + '1': { + name: 'meta.brace.square.tsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'meta.brace.square.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'numeric-literal': { + patterns: [ + { + name: 'constant.numeric.hex.tsx', + match: '\\b(?<!\\$)0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\\b(?!\\$)', + captures: { + '1': { + name: 'storage.type.numeric.bigint.tsx', + }, + }, + }, + { + name: 'constant.numeric.binary.tsx', + match: '\\b(?<!\\$)0(?:b|B)[01][01_]*(n)?\\b(?!\\$)', + captures: { + '1': { + name: 'storage.type.numeric.bigint.tsx', + }, + }, + }, + { + name: 'constant.numeric.octal.tsx', + match: '\\b(?<!\\$)0(?:o|O)?[0-7][0-7_]*(n)?\\b(?!\\$)', + captures: { + '1': { + name: 'storage.type.numeric.bigint.tsx', + }, + }, + }, + { + match: '(?x)\n(?<!\\$)(?:\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.1E+3\n (?:\\b[0-9][0-9_]*(\\.)[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1.E+3\n (?:\\B(\\.)[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # .1E+3\n (?:\\b[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*(n)?\\b)| # 1E+3\n (?:\\b[0-9][0-9_]*(\\.)[0-9][0-9_]*(n)?\\b)| # 1.1\n (?:\\b[0-9][0-9_]*(\\.)(n)?\\B)| # 1.\n (?:\\B(\\.)[0-9][0-9_]*(n)?\\b)| # .1\n (?:\\b[0-9][0-9_]*(n)?\\b(?!\\.)) # 1\n)(?!\\$)', + captures: { + '0': { + name: 'constant.numeric.decimal.tsx', + }, + '1': { + name: 'meta.delimiter.decimal.period.tsx', + }, + '2': { + name: 'storage.type.numeric.bigint.tsx', + }, + '3': { + name: 'meta.delimiter.decimal.period.tsx', + }, + '4': { + name: 'storage.type.numeric.bigint.tsx', + }, + '5': { + name: 'meta.delimiter.decimal.period.tsx', + }, + '6': { + name: 'storage.type.numeric.bigint.tsx', + }, + '7': { + name: 'storage.type.numeric.bigint.tsx', + }, + '8': { + name: 'meta.delimiter.decimal.period.tsx', + }, + '9': { + name: 'storage.type.numeric.bigint.tsx', + }, + '10': { + name: 'meta.delimiter.decimal.period.tsx', + }, + '11': { + name: 'storage.type.numeric.bigint.tsx', + }, + '12': { + name: 'meta.delimiter.decimal.period.tsx', + }, + '13': { + name: 'storage.type.numeric.bigint.tsx', + }, + '14': { + name: 'storage.type.numeric.bigint.tsx', + }, + }, + }, + ], + }, + 'boolean-literal': { + patterns: [ + { + name: 'constant.language.boolean.true.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))true(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'constant.language.boolean.false.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))false(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + ], + }, + 'null-literal': { + name: 'constant.language.null.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))null(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'this-literal': { + name: 'variable.language.this.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))this\\b(?!\\$)', + }, + 'super-literal': { + name: 'variable.language.super.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))super\\b(?!\\$)', + }, + 'undefined-literal': { + name: 'constant.language.undefined.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))undefined(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'numericConstant-literal': { + patterns: [ + { + name: 'constant.language.nan.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))NaN(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'constant.language.infinity.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Infinity(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + ], + }, + 'support-objects': { + patterns: [ + { + name: 'variable.language.arguments.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(arguments)\\b(?!\\$)', + }, + { + name: 'support.class.promise.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(Promise)\\b(?!\\$)', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(import)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(meta)\\b(?!\\$)', + captures: { + '1': { + name: 'keyword.control.import.tsx', + }, + '2': { + name: 'punctuation.accessor.tsx', + }, + '3': { + name: 'punctuation.accessor.optional.tsx', + }, + '4': { + name: 'support.variable.property.importmeta.tsx', + }, + }, + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(new)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(target)\\b(?!\\$)', + captures: { + '1': { + name: 'keyword.operator.new.tsx', + }, + '2': { + name: 'punctuation.accessor.tsx', + }, + '3': { + name: 'punctuation.accessor.optional.tsx', + }, + '4': { + name: 'support.variable.property.target.tsx', + }, + }, + }, + { + match: '(?x) (?:(\\.)|(\\?\\.(?!\\s*[[:digit:]]))) \\s* (?:\n (?:(constructor|length|prototype|__proto__)\\b(?!\\$|\\s*(<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\\())\n |\n (?:(EPSILON|MAX_SAFE_INTEGER|MAX_VALUE|MIN_SAFE_INTEGER|MIN_VALUE|NEGATIVE_INFINITY|POSITIVE_INFINITY)\\b(?!\\$)))', + captures: { + '1': { + name: 'punctuation.accessor.tsx', + }, + '2': { + name: 'punctuation.accessor.optional.tsx', + }, + '3': { + name: 'support.variable.property.tsx', + }, + '4': { + name: 'support.constant.tsx', + }, + }, + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(exports)|(module)(?:(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))(exports|id|filename|loaded|parent|children))?)\\b(?!\\$)', + captures: { + '1': { + name: 'support.type.object.module.tsx', + }, + '2': { + name: 'support.type.object.module.tsx', + }, + '3': { + name: 'punctuation.accessor.tsx', + }, + '4': { + name: 'punctuation.accessor.optional.tsx', + }, + '5': { + name: 'support.type.object.module.tsx', + }, + }, + }, + ], + }, + identifiers: { + patterns: [ + { + include: '#object-identifiers', + }, + { + match: '(?x)(?:(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*)?([_$[:alpha:]][_$[:alnum:]]*)(?=\\s*=\\s*(\n ((async\\s+)?(\n (function\\s*[(<*]) |\n (function\\s+) |\n ([_$[:alpha:]][_$[:alnum:]]*\\s*=>)\n )) |\n ((async\\s*)?(\n ((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))))) |\n # sure shot arrow functions even if => is on new line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)?\n [(]\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*\n (\n ([)]\\s*:) | # ():\n ((\\.\\.\\.\\s*)?[_$[:alpha:]][_$[:alnum:]]*\\s*:) # [(]param: | [(]...param:\n )\n) |\n(\n [<]\\s*[_$[:alpha:]][_$[:alnum:]]*\\s+extends\\s*[^=>] # < typeparam extends\n) |\n# arrow function possible to detect only with => on same line\n(\n (<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<]|\\<\\s*(((const\\s+)?[_$[:alpha:]])|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\]))([^=<>]|=[^<])*\\>)*\\>)*>\\s*)? # typeparameters\n \\(\\s*(\\/\\*([^\\*]|(\\*[^\\/]))*\\*\\/\\s*)*(([_$[:alpha:]]|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\.\\.\\.\\s*[_$[:alpha:]]))([^()\\\'\\"\\`]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))*)?\\) # parameters\n (\\s*:\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+)? # return type\n \\s*=> # arrow operator\n)\n ))\n))', + captures: { + '1': { + name: 'punctuation.accessor.tsx', + }, + '2': { + name: 'punctuation.accessor.optional.tsx', + }, + '3': { + name: 'entity.name.function.tsx', + }, + }, + }, + { + match: '(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(\\#?[[:upper:]][_$[:digit:][:upper:]]*)(?![_$[:alnum:]])', + captures: { + '1': { + name: 'punctuation.accessor.tsx', + }, + '2': { + name: 'punctuation.accessor.optional.tsx', + }, + '3': { + name: 'variable.other.constant.property.tsx', + }, + }, + }, + { + match: '(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(\\#?[_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'punctuation.accessor.tsx', + }, + '2': { + name: 'punctuation.accessor.optional.tsx', + }, + '3': { + name: 'variable.other.property.tsx', + }, + }, + }, + { + name: 'variable.other.constant.tsx', + match: '([[:upper:]][_$[:digit:][:upper:]]*)(?![_$[:alnum:]])', + }, + { + name: 'variable.other.readwrite.tsx', + match: '[_$[:alpha:]][_$[:alnum:]]*', + }, + ], + }, + 'object-identifiers': { + patterns: [ + { + name: 'support.class.tsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)(?=\\s*\\??\\.\\s*prototype\\b(?!\\$))', + }, + { + match: '(?x)(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(?:\n (\\#?[[:upper:]][_$[:digit:][:upper:]]*) |\n (\\#?[_$[:alpha:]][_$[:alnum:]]*)\n)(?=\\s*\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'punctuation.accessor.tsx', + }, + '2': { + name: 'punctuation.accessor.optional.tsx', + }, + '3': { + name: 'variable.other.constant.object.property.tsx', + }, + '4': { + name: 'variable.other.object.property.tsx', + }, + }, + }, + { + match: '(?x)(?:\n ([[:upper:]][_$[:digit:][:upper:]]*) |\n ([_$[:alpha:]][_$[:alnum:]]*)\n)(?=\\s*\\??\\.\\s*\\#?[_$[:alpha:]][_$[:alnum:]]*)', + captures: { + '1': { + name: 'variable.other.constant.object.tsx', + }, + '2': { + name: 'variable.other.object.tsx', + }, + }, + }, + ], + }, + 'type-annotation': { + patterns: [ + { + name: 'meta.type.annotation.tsx', + begin: '(:)(?=\\s*\\S)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.tsx', + }, + }, + end: '(?<![:|&])(?!\\s*[|&]\\s+)((?=^|[,);\\}\\]]|//)|(?==[^>])|((?<=[\\}>\\]\\)]|[_$[:alpha:]])\\s*(?=\\{)))', + patterns: [ + { + include: '#type', + }, + ], + }, + { + name: 'meta.type.annotation.tsx', + begin: '(:)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.tsx', + }, + }, + end: '(?<![:|&])((?=[,);\\}\\]]|\\/\\/)|(?==[^>])|(?=^\\s*$)|((?<=[\\}>\\]\\)]|[_$[:alpha:]])\\s*(?=\\{)))', + patterns: [ + { + include: '#type', + }, + ], + }, + ], + }, + 'parameter-type-annotation': { + patterns: [ + { + name: 'meta.type.annotation.tsx', + begin: '(:)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.tsx', + }, + }, + end: '(?=[,)])|(?==[^>])', + patterns: [ + { + include: '#type', + }, + ], + }, + ], + }, + 'return-type': { + patterns: [ + { + name: 'meta.return.type.tsx', + begin: '(?<=\\))\\s*(:)(?=\\s*\\S)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.tsx', + }, + }, + end: '(?<![:|&])(?=$|^|[{};,]|//)', + patterns: [ + { + include: '#return-type-core', + }, + ], + }, + { + name: 'meta.return.type.tsx', + begin: '(?<=\\))\\s*(:)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.tsx', + }, + }, + end: '(?<![:|&])((?=[{};,]|//|^\\s*$)|((?<=\\S)(?=\\s*$)))', + patterns: [ + { + include: '#return-type-core', + }, + ], + }, + ], + }, + 'return-type-core': { + patterns: [ + { + include: '#comment', + }, + { + begin: '(?<=[:|&])(?=\\s*\\{)', + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + include: '#type-predicate-operator', + }, + { + include: '#type', + }, + ], + }, + 'arrow-return-type': { + name: 'meta.return.type.arrow.tsx', + begin: '(?<=\\))\\s*(:)', + beginCaptures: { + '1': { + name: 'keyword.operator.type.annotation.tsx', + }, + }, + end: '(?==>|\\{|(^\\s*(export|function|class|interface|let|var|const|import|enum|namespace|module|type|abstract|declare)\\s+))', + patterns: [ + { + include: '#arrow-return-type-body', + }, + ], + }, + 'possibly-arrow-return-type': { + begin: '(?<=\\)|^)\\s*(:)(?=\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*=>)', + beginCaptures: { + '1': { + name: 'meta.arrow.tsx meta.return.type.arrow.tsx keyword.operator.type.annotation.tsx', + }, + }, + end: '(?==>|\\{|(^\\s*(export|function|class|interface|let|var|const|import|enum|namespace|module|type|abstract|declare)\\s+))', + contentName: 'meta.arrow.tsx meta.return.type.arrow.tsx', + patterns: [ + { + include: '#arrow-return-type-body', + }, + ], + }, + 'arrow-return-type-body': { + patterns: [ + { + begin: '(?<=[:])(?=\\s*\\{)', + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + include: '#type-predicate-operator', + }, + { + include: '#type', + }, + ], + }, + 'type-parameters': { + name: 'meta.type.parameters.tsx', + begin: '(<)', + beginCaptures: { + '1': { + name: 'punctuation.definition.typeparameters.begin.tsx', + }, + }, + end: '(>)', + endCaptures: { + '1': { + name: 'punctuation.definition.typeparameters.end.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + name: 'storage.modifier.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(extends|in|out|const)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + include: '#type', + }, + { + include: '#punctuation-comma', + }, + { + name: 'keyword.operator.assignment.tsx', + match: '(=)(?!>)', + }, + ], + }, + 'type-arguments': { + name: 'meta.type.parameters.tsx', + begin: '\\<', + beginCaptures: { + '0': { + name: 'punctuation.definition.typeparameters.begin.tsx', + }, + }, + end: '\\>', + endCaptures: { + '0': { + name: 'punctuation.definition.typeparameters.end.tsx', + }, + }, + patterns: [ + { + include: '#type-arguments-body', + }, + ], + }, + 'type-arguments-body': { + patterns: [ + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(_)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + captures: { + '0': { + name: 'keyword.operator.type.tsx', + }, + }, + }, + { + include: '#type', + }, + { + include: '#punctuation-comma', + }, + ], + }, + type: { + patterns: [ + { + include: '#comment', + }, + { + include: '#type-string', + }, + { + include: '#numeric-literal', + }, + { + include: '#type-primitive', + }, + { + include: '#type-builtin-literals', + }, + { + include: '#type-parameters', + }, + { + include: '#type-tuple', + }, + { + include: '#type-object', + }, + { + include: '#type-operators', + }, + { + include: '#type-conditional', + }, + { + include: '#type-fn-type-parameters', + }, + { + include: '#type-paren-or-function-parameters', + }, + { + include: '#type-function-return-type', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(readonly)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))\\s*', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + }, + }, + { + include: '#type-name', + }, + ], + }, + 'type-primitive': { + name: 'support.type.primitive.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(string|number|bigint|boolean|symbol|any|void|never|unknown)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'type-builtin-literals': { + name: 'support.type.builtin.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(this|true|false|undefined|null|object)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + 'type-tuple': { + name: 'meta.type.tuple.tsx', + begin: '\\[', + beginCaptures: { + '0': { + name: 'meta.brace.square.tsx', + }, + }, + end: '\\]', + endCaptures: { + '0': { + name: 'meta.brace.square.tsx', + }, + }, + patterns: [ + { + name: 'keyword.operator.rest.tsx', + match: '\\.\\.\\.', + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))([_$[:alpha:]][_$[:alnum:]]*)\\s*(\\?)?\\s*(:)', + captures: { + '1': { + name: 'entity.name.label.tsx', + }, + '2': { + name: 'keyword.operator.optional.tsx', + }, + '3': { + name: 'punctuation.separator.label.tsx', + }, + }, + }, + { + include: '#type', + }, + { + include: '#punctuation-comma', + }, + ], + }, + 'type-object': { + name: 'meta.object.type.tsx', + begin: '\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.block.tsx', + }, + }, + patterns: [ + { + include: '#comment', + }, + { + include: '#method-declaration', + }, + { + include: '#indexer-declaration', + }, + { + include: '#indexer-mapped-type-declaration', + }, + { + include: '#field-declaration', + }, + { + include: '#type-annotation', + }, + { + begin: '\\.\\.\\.', + beginCaptures: { + '0': { + name: 'keyword.operator.spread.tsx', + }, + }, + end: '(?=\\}|;|,|$)|(?<=\\})', + patterns: [ + { + include: '#type', + }, + ], + }, + { + include: '#punctuation-comma', + }, + { + include: '#punctuation-semicolon', + }, + { + include: '#type', + }, + ], + }, + 'type-conditional': { + patterns: [ + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(extends)\\s+', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + }, + end: '(?<=:)', + patterns: [ + { + begin: '\\?', + beginCaptures: { + '0': { + name: 'keyword.operator.ternary.tsx', + }, + }, + end: ':', + endCaptures: { + '0': { + name: 'keyword.operator.ternary.tsx', + }, + }, + patterns: [ + { + include: '#type', + }, + ], + }, + { + include: '#type', + }, + ], + }, + ], + }, + 'type-paren-or-function-parameters': { + name: 'meta.type.paren.cover.tsx', + begin: '\\(', + beginCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'meta.brace.round.tsx', + }, + }, + patterns: [ + { + match: '(?x)(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(public|private|protected|readonly)\\s+)?(?:(\\.\\.\\.)\\s*)?(?<!=|:)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))\\s*(\\??)(?=\\s*(:\\s*(\n (<) |\n ([(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n ))\n)) |\n(:\\s*(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))Function(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))) |\n(:\\s*((<\\s*$)|([\\(]\\s*((([\\{\\[]\\s*)?$)|((\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})\\s*((:\\s*\\{?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*)))|((\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])\\s*((:\\s*\\[?$)|((\\s*([^<>\\(\\)\\{\\}]|\\<([^<>]|\\<([^<>]|\\<[^<>]+\\>)+\\>)+\\>|\\([^\\(\\)]+\\)|\\{[^\\{\\}]+\\})+\\s*)?=\\s*))))))))', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.operator.rest.tsx', + }, + '3': { + name: 'entity.name.function.tsx variable.language.this.tsx', + }, + '4': { + name: 'entity.name.function.tsx', + }, + '5': { + name: 'keyword.operator.optional.tsx', + }, + }, + }, + { + match: '(?x)(?:(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(public|private|protected|readonly)\\s+)?(?:(\\.\\.\\.)\\s*)?(?<!=|:)(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))\\s*(\\??)(?=:)', + captures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.operator.rest.tsx', + }, + '3': { + name: 'variable.parameter.tsx variable.language.this.tsx', + }, + '4': { + name: 'variable.parameter.tsx', + }, + '5': { + name: 'keyword.operator.optional.tsx', + }, + }, + }, + { + include: '#type-annotation', + }, + { + name: 'punctuation.separator.parameter.tsx', + match: ',', + }, + { + include: '#type', + }, + ], + }, + 'type-fn-type-parameters': { + patterns: [ + { + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(abstract)\\s+)?(new)\\b(?=\\s*\\<)', + beginCaptures: { + '1': { + name: 'meta.type.constructor.tsx storage.modifier.tsx', + }, + '2': { + name: 'meta.type.constructor.tsx keyword.control.new.tsx', + }, + }, + end: '(?<=>)', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-parameters', + }, + ], + }, + { + name: 'meta.type.constructor.tsx', + begin: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(abstract)\\s+)?(new)\\b\\s*(?=\\()', + beginCaptures: { + '1': { + name: 'storage.modifier.tsx', + }, + '2': { + name: 'keyword.control.new.tsx', + }, + }, + end: '(?<=\\))', + patterns: [ + { + include: '#function-parameters', + }, + ], + }, + { + name: 'meta.type.function.tsx', + begin: '(?x)(\n (?=\n [(]\\s*(\n ([)]) |\n (\\.\\.\\.) |\n ([_$[:alnum:]]+\\s*(\n ([:,?=])|\n ([)]\\s*=>)\n ))\n )\n )\n)', + end: '(?<=\\))', + patterns: [ + { + include: '#function-parameters', + }, + ], + }, + ], + }, + 'type-function-return-type': { + patterns: [ + { + name: 'meta.type.function.return.tsx', + begin: '(=>)(?=\\s*\\S)', + beginCaptures: { + '1': { + name: 'storage.type.function.arrow.tsx', + }, + }, + end: '(?<!=>)(?<![|&])(?=[,\\]\\)\\{\\}=;>:\\?]|//|$)', + patterns: [ + { + include: '#type-function-return-type-core', + }, + ], + }, + { + name: 'meta.type.function.return.tsx', + begin: '=>', + beginCaptures: { + '0': { + name: 'storage.type.function.arrow.tsx', + }, + }, + end: '(?<!=>)(?<![|&])((?=[,\\]\\)\\{\\}=;:\\?>]|//|^\\s*$)|((?<=\\S)(?=\\s*$)))', + patterns: [ + { + include: '#type-function-return-type-core', + }, + ], + }, + ], + }, + 'type-function-return-type-core': { + patterns: [ + { + include: '#comment', + }, + { + begin: '(?<==>)(?=\\s*\\{)', + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + include: '#type-predicate-operator', + }, + { + include: '#type', + }, + ], + }, + 'type-operators': { + patterns: [ + { + include: '#typeof-operator', + }, + { + include: '#type-infer', + }, + { + begin: '([&|])(?=\\s*\\{)', + beginCaptures: { + '0': { + name: 'keyword.operator.type.tsx', + }, + }, + end: '(?<=\\})', + patterns: [ + { + include: '#type-object', + }, + ], + }, + { + begin: '[&|]', + beginCaptures: { + '0': { + name: 'keyword.operator.type.tsx', + }, + }, + end: '(?=\\S)', + }, + { + name: 'keyword.operator.expression.keyof.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))keyof(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.operator.ternary.tsx', + match: '(\\?|\\:)', + }, + { + name: 'keyword.operator.expression.import.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))import(?=\\s*\\()', + }, + ], + }, + 'type-infer': { + patterns: [ + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(infer)\\s+([_$[:alpha:]][_$[:alnum:]]*)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))(?:\\s+(extends)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.)))?', + name: 'meta.type.infer.tsx', + captures: { + '1': { + name: 'keyword.operator.expression.infer.tsx', + }, + '2': { + name: 'entity.name.type.tsx', + }, + '3': { + name: 'keyword.operator.expression.extends.tsx', + }, + }, + }, + ], + }, + 'type-predicate-operator': { + patterns: [ + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(?:(asserts)\\s+)?(?!asserts)(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))\\s(is)(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + captures: { + '1': { + name: 'keyword.operator.type.asserts.tsx', + }, + '2': { + name: 'variable.parameter.tsx variable.language.this.tsx', + }, + '3': { + name: 'variable.parameter.tsx', + }, + '4': { + name: 'keyword.operator.expression.is.tsx', + }, + }, + }, + { + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))(asserts)\\s+(?!is)(?:(this)|([_$[:alpha:]][_$[:alnum:]]*))(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + captures: { + '1': { + name: 'keyword.operator.type.asserts.tsx', + }, + '2': { + name: 'variable.parameter.tsx variable.language.this.tsx', + }, + '3': { + name: 'variable.parameter.tsx', + }, + }, + }, + { + name: 'keyword.operator.type.asserts.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))asserts(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + { + name: 'keyword.operator.expression.is.tsx', + match: '(?<![_$[:alnum:]])(?:(?<=\\.\\.\\.)|(?<!\\.))is(?![_$[:alnum:]])(?:(?=\\.\\.\\.)|(?!\\.))', + }, + ], + }, + 'type-name': { + patterns: [ + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))\\s*(<)', + captures: { + '1': { + name: 'entity.name.type.module.tsx', + }, + '2': { + name: 'punctuation.accessor.tsx', + }, + '3': { + name: 'punctuation.accessor.optional.tsx', + }, + '4': { + name: 'meta.type.parameters.tsx punctuation.definition.typeparameters.begin.tsx', + }, + }, + end: '(>)', + endCaptures: { + '1': { + name: 'meta.type.parameters.tsx punctuation.definition.typeparameters.end.tsx', + }, + }, + contentName: 'meta.type.parameters.tsx', + patterns: [ + { + include: '#type-arguments-body', + }, + ], + }, + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(<)', + beginCaptures: { + '1': { + name: 'entity.name.type.tsx', + }, + '2': { + name: 'meta.type.parameters.tsx punctuation.definition.typeparameters.begin.tsx', + }, + }, + end: '(>)', + endCaptures: { + '1': { + name: 'meta.type.parameters.tsx punctuation.definition.typeparameters.end.tsx', + }, + }, + contentName: 'meta.type.parameters.tsx', + patterns: [ + { + include: '#type-arguments-body', + }, + ], + }, + { + match: '([_$[:alpha:]][_$[:alnum:]]*)\\s*(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))', + captures: { + '1': { + name: 'entity.name.type.module.tsx', + }, + '2': { + name: 'punctuation.accessor.tsx', + }, + '3': { + name: 'punctuation.accessor.optional.tsx', + }, + }, + }, + { + name: 'entity.name.type.tsx', + match: '[_$[:alpha:]][_$[:alnum:]]*', + }, + ], + }, + 'punctuation-comma': { + name: 'punctuation.separator.comma.tsx', + match: ',', + }, + 'punctuation-semicolon': { + name: 'punctuation.terminator.statement.tsx', + match: ';', + }, + 'punctuation-accessor': { + match: '(?:(\\.)|(\\?\\.(?!\\s*[[:digit:]])))', + captures: { + '1': { + name: 'punctuation.accessor.tsx', + }, + '2': { + name: 'punctuation.accessor.optional.tsx', + }, + }, + }, + string: { + patterns: [ + { + include: '#qstring-single', + }, + { + include: '#qstring-double', + }, + { + include: '#template', + }, + ], + }, + 'qstring-double': { + name: 'string.quoted.double.tsx', + begin: '"', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + end: '(")|((?:[^\\\\\\n])$)', + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.tsx', + }, + '2': { + name: 'invalid.illegal.newline.tsx', + }, + }, + patterns: [ + { + include: '#string-character-escape', + }, + ], + }, + 'qstring-single': { + name: 'string.quoted.single.tsx', + begin: `'`, + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + end: `(\\')|((?:[^\\\\\\n])$)`, + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.tsx', + }, + '2': { + name: 'invalid.illegal.newline.tsx', + }, + }, + patterns: [ + { + include: '#string-character-escape', + }, + ], + }, + 'string-character-escape': { + name: 'constant.character.escape.tsx', + match: '\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)', + }, + template: { + patterns: [ + { + include: '#template-call', + }, + { + contentName: 'string.template.tsx', + begin: '([_$[:alpha:]][_$[:alnum:]]*)?(`)', + beginCaptures: { + '1': { + name: 'entity.name.function.tagged-template.tsx', + }, + '2': { + name: 'string.template.tsx punctuation.definition.string.template.begin.tsx', + }, + }, + end: '`', + endCaptures: { + '0': { + name: 'string.template.tsx punctuation.definition.string.template.end.tsx', + }, + }, + patterns: [ + { + include: '#template-substitution-element', + }, + { + include: '#string-character-escape', + }, + ], + }, + ], + }, + 'template-call': { + patterns: [ + { + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*\\s*\\??\\.\\s*)*|(\\??\\.\\s*)?)([_$[:alpha:]][_$[:alnum:]]*)(<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?<!=)\\>))*(?<!=)\\>)*(?<!=)>\\s*)?`)', + end: '(?=`)', + patterns: [ + { + begin: '(?=(([_$[:alpha:]][_$[:alnum:]]*\\s*\\??\\.\\s*)*|(\\??\\.\\s*)?)([_$[:alpha:]][_$[:alnum:]]*))', + end: '(?=(<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?<!=)\\>))*(?<!=)\\>)*(?<!=)>\\s*)?`)', + patterns: [ + { + include: '#support-function-call-identifiers', + }, + { + name: 'entity.name.function.tagged-template.tsx', + match: '([_$[:alpha:]][_$[:alnum:]]*)', + }, + ], + }, + { + include: '#type-arguments', + }, + ], + }, + { + begin: '([_$[:alpha:]][_$[:alnum:]]*)?\\s*(?=(<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))(([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>|\\<\\s*(((keyof|infer|typeof|readonly)\\s+)|(([_$[:alpha:]][_$[:alnum:]]*|(\\{([^\\{\\}]|(\\{([^\\{\\}]|\\{[^\\{\\}]*\\})*\\}))*\\})|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(\\[([^\\[\\]]|(\\[([^\\[\\]]|\\[[^\\[\\]]*\\])*\\]))*\\])|(\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`))(?=\\s*([\\<\\>\\,\\.\\[]|=>|&(?!&)|\\|(?!\\|)))))([^<>\\(]|(\\(([^\\(\\)]|(\\(([^\\(\\)]|\\([^\\(\\)]*\\))*\\)))*\\))|(?<==)\\>)*(?<!=)\\>))*(?<!=)\\>)*(?<!=)>\\s*)`)', + beginCaptures: { + '1': { + name: 'entity.name.function.tagged-template.tsx', + }, + }, + end: '(?=`)', + patterns: [ + { + include: '#type-arguments', + }, + ], + }, + ], + }, + 'template-substitution-element': { + name: 'meta.template.expression.tsx', + begin: '\\$\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.template-expression.begin.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.template-expression.end.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + contentName: 'meta.embedded.line.tsx', + }, + 'type-string': { + patterns: [ + { + include: '#qstring-single', + }, + { + include: '#qstring-double', + }, + { + include: '#template-type', + }, + ], + }, + 'template-type': { + patterns: [ + { + include: '#template-call', + }, + { + contentName: 'string.template.tsx', + begin: '([_$[:alpha:]][_$[:alnum:]]*)?(`)', + beginCaptures: { + '1': { + name: 'entity.name.function.tagged-template.tsx', + }, + '2': { + name: 'string.template.tsx punctuation.definition.string.template.begin.tsx', + }, + }, + end: '`', + endCaptures: { + '0': { + name: 'string.template.tsx punctuation.definition.string.template.end.tsx', + }, + }, + patterns: [ + { + include: '#template-type-substitution-element', + }, + { + include: '#string-character-escape', + }, + ], + }, + ], + }, + 'template-type-substitution-element': { + name: 'meta.template.expression.tsx', + begin: '\\$\\{', + beginCaptures: { + '0': { + name: 'punctuation.definition.template-expression.begin.tsx', + }, + }, + end: '\\}', + endCaptures: { + '0': { + name: 'punctuation.definition.template-expression.end.tsx', + }, + }, + patterns: [ + { + include: '#type', + }, + ], + contentName: 'meta.embedded.line.tsx', + }, + regex: { + patterns: [ + { + name: 'string.regexp.tsx', + begin: '(?<!\\+\\+|--|})(?<=[=(:,\\[?+!]|^return|[^\\._$[:alnum:]]return|^case|[^\\._$[:alnum:]]case|=>|&&|\\|\\||\\*\\/)\\s*(\\/)(?![\\/*])(?=(?:[^\\/\\\\\\[\\()]|\\\\.|\\[([^\\]\\\\]|\\\\.)+\\]|\\(([^\\)\\\\]|\\\\.)+\\))+\\/([dgimsuy]+|(?![\\/\\*])|(?=\\/\\*))(?!\\s*[a-zA-Z0-9_$]))', + beginCaptures: { + '1': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + end: '(/)([dgimsuy]*)', + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.tsx', + }, + '2': { + name: 'keyword.other.tsx', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + { + name: 'string.regexp.tsx', + begin: '((?<![_$[:alnum:])\\]]|\\+\\+|--|}|\\*\\/)|((?<=^return|[^\\._$[:alnum:]]return|^case|[^\\._$[:alnum:]]case))\\s*)\\/(?![\\/*])(?=(?:[^\\/\\\\\\[]|\\\\.|\\[([^\\]\\\\]|\\\\.)*\\])+\\/([dgimsuy]+|(?![\\/\\*])|(?=\\/\\*))(?!\\s*[a-zA-Z0-9_$]))', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + end: '(/)([dgimsuy]*)', + endCaptures: { + '1': { + name: 'punctuation.definition.string.end.tsx', + }, + '2': { + name: 'keyword.other.tsx', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + ], + }, + regexp: { + patterns: [ + { + name: 'keyword.control.anchor.regexp', + match: '\\\\[bB]|\\^|\\$', + }, + { + match: '\\\\[1-9]\\d*|\\\\k<([a-zA-Z_$][\\w$]*)>', + captures: { + '0': { + name: 'keyword.other.back-reference.regexp', + }, + '1': { + name: 'variable.other.regexp', + }, + }, + }, + { + name: 'keyword.operator.quantifier.regexp', + match: '[?+*]|\\{(\\d+,\\d+|\\d+,|,\\d+|\\d+)\\}\\??', + }, + { + name: 'keyword.operator.or.regexp', + match: '\\|', + }, + { + name: 'meta.group.assertion.regexp', + begin: '(\\()((\\?=)|(\\?!)|(\\?<=)|(\\?<!))', + beginCaptures: { + '1': { + name: 'punctuation.definition.group.regexp', + }, + '2': { + name: 'punctuation.definition.group.assertion.regexp', + }, + '3': { + name: 'meta.assertion.look-ahead.regexp', + }, + '4': { + name: 'meta.assertion.negative-look-ahead.regexp', + }, + '5': { + name: 'meta.assertion.look-behind.regexp', + }, + '6': { + name: 'meta.assertion.negative-look-behind.regexp', + }, + }, + end: '(\\))', + endCaptures: { + '1': { + name: 'punctuation.definition.group.regexp', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + { + name: 'meta.group.regexp', + begin: '\\((?:(\\?:)|(?:\\?<([a-zA-Z_$][\\w$]*)>))?', + beginCaptures: { + '0': { + name: 'punctuation.definition.group.regexp', + }, + '1': { + name: 'punctuation.definition.group.no-capture.regexp', + }, + '2': { + name: 'variable.other.regexp', + }, + }, + end: '\\)', + endCaptures: { + '0': { + name: 'punctuation.definition.group.regexp', + }, + }, + patterns: [ + { + include: '#regexp', + }, + ], + }, + { + name: 'constant.other.character-class.set.regexp', + begin: '(\\[)(\\^)?', + beginCaptures: { + '1': { + name: 'punctuation.definition.character-class.regexp', + }, + '2': { + name: 'keyword.operator.negation.regexp', + }, + }, + end: '(\\])', + endCaptures: { + '1': { + name: 'punctuation.definition.character-class.regexp', + }, + }, + patterns: [ + { + name: 'constant.other.character-class.range.regexp', + match: '(?:.|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))', + captures: { + '1': { + name: 'constant.character.numeric.regexp', + }, + '2': { + name: 'constant.character.control.regexp', + }, + '3': { + name: 'constant.character.escape.backslash.regexp', + }, + '4': { + name: 'constant.character.numeric.regexp', + }, + '5': { + name: 'constant.character.control.regexp', + }, + '6': { + name: 'constant.character.escape.backslash.regexp', + }, + }, + }, + { + include: '#regex-character-class', + }, + ], + }, + { + include: '#regex-character-class', + }, + ], + }, + 'regex-character-class': { + patterns: [ + { + name: 'constant.other.character-class.regexp', + match: '\\\\[wWsSdDtrnvf]|\\.', + }, + { + name: 'constant.character.numeric.regexp', + match: '\\\\([0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4})', + }, + { + name: 'constant.character.control.regexp', + match: '\\\\c[A-Z]', + }, + { + name: 'constant.character.escape.backslash.regexp', + match: '\\\\.', + }, + ], + }, + comment: { + patterns: [ + { + name: 'comment.block.documentation.tsx', + begin: '/\\*\\*(?!/)', + beginCaptures: { + '0': { + name: 'punctuation.definition.comment.tsx', + }, + }, + end: '\\*/', + endCaptures: { + '0': { + name: 'punctuation.definition.comment.tsx', + }, + }, + patterns: [ + { + include: '#docblock', + }, + ], + }, + { + name: 'comment.block.tsx', + begin: '(/\\*)(?:\\s*((@)internal)(?=\\s|(\\*/)))?', + beginCaptures: { + '1': { + name: 'punctuation.definition.comment.tsx', + }, + '2': { + name: 'storage.type.internaldeclaration.tsx', + }, + '3': { + name: 'punctuation.decorator.internaldeclaration.tsx', + }, + }, + end: '\\*/', + endCaptures: { + '0': { + name: 'punctuation.definition.comment.tsx', + }, + }, + }, + { + begin: '(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)', + beginCaptures: { + '1': { + name: 'punctuation.whitespace.comment.leading.tsx', + }, + '2': { + name: 'comment.line.double-slash.tsx', + }, + '3': { + name: 'punctuation.definition.comment.tsx', + }, + '4': { + name: 'storage.type.internaldeclaration.tsx', + }, + '5': { + name: 'punctuation.decorator.internaldeclaration.tsx', + }, + }, + end: '(?=$)', + contentName: 'comment.line.double-slash.tsx', + }, + ], + }, + 'single-line-comment-consuming-line-ending': { + begin: '(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)', + beginCaptures: { + '1': { + name: 'punctuation.whitespace.comment.leading.tsx', + }, + '2': { + name: 'comment.line.double-slash.tsx', + }, + '3': { + name: 'punctuation.definition.comment.tsx', + }, + '4': { + name: 'storage.type.internaldeclaration.tsx', + }, + '5': { + name: 'punctuation.decorator.internaldeclaration.tsx', + }, + }, + end: '(?=^)', + contentName: 'comment.line.double-slash.tsx', + }, + directives: { + name: 'comment.line.triple-slash.directive.tsx', + begin: '^(///)\\s*(?=<(reference|amd-dependency|amd-module)(\\s+(path|types|no-default-lib|lib|name|resolution-mode)\\s*=\\s*((\\\'([^\\\'\\\\]|\\\\.)*\\\')|(\\"([^\\"\\\\]|\\\\.)*\\")|(\\`([^\\`\\\\]|\\\\.)*\\`)))+\\s*/>\\s*$)', + beginCaptures: { + '1': { + name: 'punctuation.definition.comment.tsx', + }, + }, + end: '(?=$)', + patterns: [ + { + name: 'meta.tag.tsx', + begin: '(<)(reference|amd-dependency|amd-module)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.directive.tsx', + }, + '2': { + name: 'entity.name.tag.directive.tsx', + }, + }, + end: '/>', + endCaptures: { + '0': { + name: 'punctuation.definition.tag.directive.tsx', + }, + }, + patterns: [ + { + name: 'entity.other.attribute-name.directive.tsx', + match: 'path|types|no-default-lib|lib|name|resolution-mode', + }, + { + name: 'keyword.operator.assignment.tsx', + match: '=', + }, + { + include: '#string', + }, + ], + }, + ], + }, + docblock: { + patterns: [ + { + match: '(?x)\n((@)(?:access|api))\n\\s+\n(private|protected|public)\n\\b', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'constant.language.access-type.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)author)\n\\s+\n(\n [^@\\s<>*/]\n (?:[^@<>*/]|\\*[^/])*\n)\n(?:\n \\s*\n (<)\n ([^>\\s]+)\n (>)\n)?', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'entity.name.type.instance.jsdoc', + }, + '4': { + name: 'punctuation.definition.bracket.angle.begin.jsdoc', + }, + '5': { + name: 'constant.other.email.link.underline.jsdoc', + }, + '6': { + name: 'punctuation.definition.bracket.angle.end.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)borrows) \\s+\n((?:[^@\\s*/]|\\*[^/])+) # <that namepath>\n\\s+ (as) \\s+ # as\n((?:[^@\\s*/]|\\*[^/])+) # <this namepath>', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'entity.name.type.instance.jsdoc', + }, + '4': { + name: 'keyword.operator.control.jsdoc', + }, + '5': { + name: 'entity.name.type.instance.jsdoc', + }, + }, + }, + { + name: 'meta.example.jsdoc', + begin: '((@)example)\\s+', + end: '(?=@|\\*/)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + patterns: [ + { + match: '^\\s\\*\\s+', + }, + { + contentName: 'constant.other.description.jsdoc', + begin: '\\G(<)caption(>)', + beginCaptures: { + '0': { + name: 'entity.name.tag.inline.jsdoc', + }, + '1': { + name: 'punctuation.definition.bracket.angle.begin.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.angle.end.jsdoc', + }, + }, + end: '(</)caption(>)|(?=\\*/)', + endCaptures: { + '0': { + name: 'entity.name.tag.inline.jsdoc', + }, + '1': { + name: 'punctuation.definition.bracket.angle.begin.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.angle.end.jsdoc', + }, + }, + }, + { + match: '[^\\s@*](?:[^*]|\\*[^/])*', + captures: { + '0': { + name: 'source.embedded.tsx', + }, + }, + }, + ], + }, + { + match: '(?x) ((@)kind) \\s+ (class|constant|event|external|file|function|member|mixin|module|namespace|typedef) \\b', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'constant.language.symbol-type.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)see)\n\\s+\n(?:\n # URL\n (\n (?=https?://)\n (?:[^\\s*]|\\*[^/])+\n )\n |\n # JSDoc namepath\n (\n (?!\n # Avoid matching bare URIs (also acceptable as links)\n https?://\n |\n # Avoid matching {@inline tags}; we match those below\n (?:\\[[^\\[\\]]*\\])? # Possible description [preceding]{@tag}\n {@(?:link|linkcode|linkplain|tutorial)\\b\n )\n # Matched namepath\n (?:[^@\\s*/]|\\*[^/])+\n )\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.link.underline.jsdoc', + }, + '4': { + name: 'entity.name.type.instance.jsdoc', + }, + }, + }, + { + match: '(?x)\n((@)template)\n\\s+\n# One or more valid identifiers\n(\n [A-Za-z_$] # First character: non-numeric word character\n [\\w$.\\[\\]]* # Rest of identifier\n (?: # Possible list of additional identifiers\n \\s* , \\s*\n [A-Za-z_$]\n [\\w$.\\[\\]]*\n )*\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + }, + }, + { + begin: '(?x)((@)template)\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + { + name: 'variable.other.jsdoc', + match: '([A-Za-z_$][\\w$.\\[\\]]*)', + }, + ], + }, + { + match: '(?x)\n(\n (@)\n (?:arg|argument|const|constant|member|namespace|param|var)\n)\n\\s+\n(\n [A-Za-z_$]\n [\\w$.\\[\\]]*\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + }, + }, + { + begin: '((@)typedef)\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + { + name: 'entity.name.type.instance.jsdoc', + match: '(?:[^@\\s*/]|\\*[^/])+', + }, + ], + }, + { + begin: '((@)(?:arg|argument|const|constant|member|namespace|param|prop|property|var))\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + { + name: 'variable.other.jsdoc', + match: '([A-Za-z_$][\\w$.\\[\\]]*)', + }, + { + name: 'variable.other.jsdoc', + match: '(?x)\n(\\[)\\s*\n[\\w$]+\n(?:\n (?:\\[\\])? # Foo[ ].bar properties within an array\n \\. # Foo.Bar namespaced parameter\n [\\w$]+\n)*\n(?:\n \\s*\n (=) # [foo=bar] Default parameter value\n \\s*\n (\n # The inner regexes are to stop the match early at */ and to not stop at escaped quotes\n (?>\n "(?:(?:\\*(?!/))|(?:\\\\(?!"))|[^*\\\\])*?" | # [foo="bar"] Double-quoted\n \'(?:(?:\\*(?!/))|(?:\\\\(?!\'))|[^*\\\\])*?\' | # [foo=\'bar\'] Single-quoted\n \\[ (?:(?:\\*(?!/))|[^*])*? \\] | # [foo=[1,2]] Array literal\n (?:(?:\\*(?!/))|\\s(?!\\s*\\])|\\[.*?(?:\\]|(?=\\*/))|[^*\\s\\[\\]])* # Everything else\n )*\n )\n)?\n\\s*(?:(\\])((?:[^*\\s]|\\*[^\\s/])+)?|(?=\\*/))', + captures: { + '1': { + name: 'punctuation.definition.optional-value.begin.bracket.square.jsdoc', + }, + '2': { + name: 'keyword.operator.assignment.jsdoc', + }, + '3': { + name: 'source.embedded.tsx', + }, + '4': { + name: 'punctuation.definition.optional-value.end.bracket.square.jsdoc', + }, + '5': { + name: 'invalid.illegal.syntax.jsdoc', + }, + }, + }, + ], + }, + { + begin: '(?x)\n(\n (@)\n (?:define|enum|exception|export|extends|lends|implements|modifies\n |namespace|private|protected|returns?|satisfies|suppress|this|throws|type\n |yields?)\n)\n\\s+(?={)', + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + end: '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])', + patterns: [ + { + include: '#jsdoctype', + }, + ], + }, + { + match: '(?x)\n(\n (@)\n (?:alias|augments|callback|constructs|emits|event|fires|exports?\n |extends|external|function|func|host|lends|listens|interface|memberof!?\n |method|module|mixes|mixin|name|requires|see|this|typedef|uses)\n)\n\\s+\n(\n (?:\n [^{}@\\s*] | \\*[^/]\n )+\n)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'entity.name.type.instance.jsdoc', + }, + }, + }, + { + contentName: 'variable.other.jsdoc', + begin: `((@)(?:default(?:value)?|license|version))\\s+(([''"]))`, + beginCaptures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + '4': { + name: 'punctuation.definition.string.begin.jsdoc', + }, + }, + end: '(\\3)|(?=$|\\*/)', + endCaptures: { + '0': { + name: 'variable.other.jsdoc', + }, + '1': { + name: 'punctuation.definition.string.end.jsdoc', + }, + }, + }, + { + match: '((@)(?:default(?:value)?|license|tutorial|variation|version))\\s+([^\\s*]+)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + '3': { + name: 'variable.other.jsdoc', + }, + }, + }, + { + name: 'storage.type.class.jsdoc', + match: '(?x) (@) (?:abstract|access|alias|api|arg|argument|async|attribute|augments|author|beta|borrows|bubbles |callback|chainable|class|classdesc|code|config|const|constant|constructor|constructs|copyright |default|defaultvalue|define|deprecated|desc|description|dict|emits|enum|event|example|exception |exports?|extends|extension(?:_?for)?|external|externs|file|fileoverview|final|fires|for|func |function|generator|global|hideconstructor|host|ignore|implements|implicitCast|inherit[Dd]oc |inner|instance|interface|internal|kind|lends|license|listens|main|member|memberof!?|method |mixes|mixins?|modifies|module|name|namespace|noalias|nocollapse|nocompile|nosideeffects |override|overview|package|param|polymer(?:Behavior)?|preserve|private|prop|property|protected |public|read[Oo]nly|record|require[ds]|returns?|see|since|static|struct|submodule|summary |suppress|template|this|throws|todo|tutorial|type|typedef|unrestricted|uses|var|variation |version|virtual|writeOnce|yields?) \\b', + captures: { + '1': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + }, + { + include: '#inline-tags', + }, + { + match: '((@)(?:[_$[:alpha:]][_$[:alnum:]]*))(?=\\s+)', + captures: { + '1': { + name: 'storage.type.class.jsdoc', + }, + '2': { + name: 'punctuation.definition.block.tag.jsdoc', + }, + }, + }, + ], + }, + brackets: { + patterns: [ + { + begin: '{', + end: '}|(?=\\*/)', + patterns: [ + { + include: '#brackets', + }, + ], + }, + { + begin: '\\[', + end: '\\]|(?=\\*/)', + patterns: [ + { + include: '#brackets', + }, + ], + }, + ], + }, + 'inline-tags': { + patterns: [ + { + name: 'constant.other.description.jsdoc', + match: '(\\[)[^\\]]+(\\])(?={@(?:link|linkcode|linkplain|tutorial))', + captures: { + '1': { + name: 'punctuation.definition.bracket.square.begin.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.square.end.jsdoc', + }, + }, + }, + { + name: 'entity.name.type.instance.jsdoc', + begin: '({)((@)(?:link(?:code|plain)?|tutorial))\\s*', + beginCaptures: { + '1': { + name: 'punctuation.definition.bracket.curly.begin.jsdoc', + }, + '2': { + name: 'storage.type.class.jsdoc', + }, + '3': { + name: 'punctuation.definition.inline.tag.jsdoc', + }, + }, + end: '}|(?=\\*/)', + endCaptures: { + '0': { + name: 'punctuation.definition.bracket.curly.end.jsdoc', + }, + }, + patterns: [ + { + match: '\\G((?=https?://)(?:[^|}\\s*]|\\*[/])+)(\\|)?', + captures: { + '1': { + name: 'variable.other.link.underline.jsdoc', + }, + '2': { + name: 'punctuation.separator.pipe.jsdoc', + }, + }, + }, + { + match: '\\G((?:[^{}@\\s|*]|\\*[^/])+)(\\|)?', + captures: { + '1': { + name: 'variable.other.description.jsdoc', + }, + '2': { + name: 'punctuation.separator.pipe.jsdoc', + }, + }, + }, + ], + }, + ], + }, + jsdoctype: { + patterns: [ + { + contentName: 'entity.name.type.instance.jsdoc', + begin: '\\G({)', + beginCaptures: { + '0': { + name: 'entity.name.type.instance.jsdoc', + }, + '1': { + name: 'punctuation.definition.bracket.curly.begin.jsdoc', + }, + }, + end: '((}))\\s*|(?=\\*/)', + endCaptures: { + '1': { + name: 'entity.name.type.instance.jsdoc', + }, + '2': { + name: 'punctuation.definition.bracket.curly.end.jsdoc', + }, + }, + patterns: [ + { + include: '#brackets', + }, + ], + }, + ], + }, + jsx: { + patterns: [ + { + include: '#jsx-tag-without-attributes-in-expression', + }, + { + include: '#jsx-tag-in-expression', + }, + ], + }, + 'jsx-tag-without-attributes-in-expression': { + begin: '(?<!\\+\\+|--)(?<=[({\\[,?=>:*]|&&|\\|\\||\\?|\\*\\/|^await|[^\\._$[:alnum:]]await|^return|[^\\._$[:alnum:]]return|^default|[^\\._$[:alnum:]]default|^yield|[^\\._$[:alnum:]]yield|^)\\s*(?=(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))?\\s*(>))', + end: '(?!(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))?\\s*(>))', + patterns: [ + { + include: '#jsx-tag-without-attributes', + }, + ], + }, + 'jsx-tag-without-attributes': { + name: 'meta.tag.without-attributes.tsx', + begin: '(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))?\\s*(>)', + end: '(</)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))?\\s*(>)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.begin.tsx', + }, + '2': { + name: 'entity.name.tag.namespace.tsx', + }, + '3': { + name: 'punctuation.separator.namespace.tsx', + }, + '4': { + name: 'entity.name.tag.tsx', + }, + '5': { + name: 'support.class.component.tsx', + }, + '6': { + name: 'punctuation.definition.tag.end.tsx', + }, + }, + endCaptures: { + '1': { + name: 'punctuation.definition.tag.begin.tsx', + }, + '2': { + name: 'entity.name.tag.namespace.tsx', + }, + '3': { + name: 'punctuation.separator.namespace.tsx', + }, + '4': { + name: 'entity.name.tag.tsx', + }, + '5': { + name: 'support.class.component.tsx', + }, + '6': { + name: 'punctuation.definition.tag.end.tsx', + }, + }, + contentName: 'meta.jsx.children.tsx', + patterns: [ + { + include: '#jsx-children', + }, + ], + }, + 'jsx-tag-in-expression': { + begin: '(?x)\n (?<!\\+\\+|--)(?<=[({\\[,?=>:*]|&&|\\|\\||\\?|\\*\\/|^await|[^\\._$[:alnum:]]await|^return|[^\\._$[:alnum:]]return|^default|[^\\._$[:alnum:]]default|^yield|[^\\._$[:alnum:]]yield|^)\\s*\n (?!<\\s*[_$[:alpha:]][_$[:alnum:]]*((\\s+extends\\s+[^=>])|,)) # look ahead is not type parameter of arrow\n (?=(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))(?=((<\\s*)|(\\s+))(?!\\?)|\\/?>))', + end: '(?!(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))(?=((<\\s*)|(\\s+))(?!\\?)|\\/?>))', + patterns: [ + { + include: '#jsx-tag', + }, + ], + }, + 'jsx-tag': { + name: 'meta.tag.tsx', + begin: '(?=(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))(?=((<\\s*)|(\\s+))(?!\\?)|\\/?>))', + end: '(/>)|(?:(</)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))?\\s*(>))', + endCaptures: { + '1': { + name: 'punctuation.definition.tag.end.tsx', + }, + '2': { + name: 'punctuation.definition.tag.begin.tsx', + }, + '3': { + name: 'entity.name.tag.namespace.tsx', + }, + '4': { + name: 'punctuation.separator.namespace.tsx', + }, + '5': { + name: 'entity.name.tag.tsx', + }, + '6': { + name: 'support.class.component.tsx', + }, + '7': { + name: 'punctuation.definition.tag.end.tsx', + }, + }, + patterns: [ + { + begin: '(<)\\s*(?:([_$[:alpha:]][-_$[:alnum:].]*)(?<!\\.|-)(:))?((?:[a-z][a-z0-9]*|([_$[:alpha:]][-_$[:alnum:].]*))(?<!\\.|-))(?=((<\\s*)|(\\s+))(?!\\?)|\\/?>)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.begin.tsx', + }, + '2': { + name: 'entity.name.tag.namespace.tsx', + }, + '3': { + name: 'punctuation.separator.namespace.tsx', + }, + '4': { + name: 'entity.name.tag.tsx', + }, + '5': { + name: 'support.class.component.tsx', + }, + }, + end: '(?=[/]?>)', + patterns: [ + { + include: '#comment', + }, + { + include: '#type-arguments', + }, + { + include: '#jsx-tag-attributes', + }, + ], + }, + { + begin: '(>)', + beginCaptures: { + '1': { + name: 'punctuation.definition.tag.end.tsx', + }, + }, + end: '(?=</)', + contentName: 'meta.jsx.children.tsx', + patterns: [ + { + include: '#jsx-children', + }, + ], + }, + ], + }, + 'jsx-children': { + patterns: [ + { + include: '#jsx-tag-without-attributes', + }, + { + include: '#jsx-tag', + }, + { + include: '#jsx-evaluated-code', + }, + { + include: '#jsx-entities', + }, + ], + }, + 'jsx-evaluated-code': { + contentName: 'meta.embedded.expression.tsx', + begin: '\\{', + end: '\\}', + beginCaptures: { + '0': { + name: 'punctuation.section.embedded.begin.tsx', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.section.embedded.end.tsx', + }, + }, + patterns: [ + { + include: '#expression', + }, + ], + }, + 'jsx-entities': { + patterns: [ + { + name: 'constant.character.entity.tsx', + match: '(&)([a-zA-Z0-9]+|#[0-9]+|#x[0-9a-fA-F]+)(;)', + captures: { + '1': { + name: 'punctuation.definition.entity.tsx', + }, + '3': { + name: 'punctuation.definition.entity.tsx', + }, + }, + }, + ], + }, + 'jsx-tag-attributes': { + name: 'meta.tag.attributes.tsx', + begin: '\\s+', + end: '(?=[/]?>)', + patterns: [ + { + include: '#comment', + }, + { + include: '#jsx-tag-attribute-name', + }, + { + include: '#jsx-tag-attribute-assignment', + }, + { + include: '#jsx-string-double-quoted', + }, + { + include: '#jsx-string-single-quoted', + }, + { + include: '#jsx-evaluated-code', + }, + { + include: '#jsx-tag-attributes-illegal', + }, + ], + }, + 'jsx-tag-attribute-name': { + match: '(?x)\n \\s*\n (?:([_$[:alpha:]][-_$[:alnum:].]*)(:))?\n ([_$[:alpha:]][-_$[:alnum:]]*)\n (?=\\s|=|/?>|/\\*|//)', + captures: { + '1': { + name: 'entity.other.attribute-name.namespace.tsx', + }, + '2': { + name: 'punctuation.separator.namespace.tsx', + }, + '3': { + name: 'entity.other.attribute-name.tsx', + }, + }, + }, + 'jsx-tag-attribute-assignment': { + name: 'keyword.operator.assignment.tsx', + match: '=(?=\\s*(?:\'|"|{|/\\*|//|\\n))', + }, + 'jsx-string-double-quoted': { + name: 'string.quoted.double.tsx', + begin: '"', + end: '"', + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.string.end.tsx', + }, + }, + patterns: [ + { + include: '#jsx-entities', + }, + ], + }, + 'jsx-string-single-quoted': { + name: 'string.quoted.single.tsx', + begin: `'`, + end: `'`, + beginCaptures: { + '0': { + name: 'punctuation.definition.string.begin.tsx', + }, + }, + endCaptures: { + '0': { + name: 'punctuation.definition.string.end.tsx', + }, + }, + patterns: [ + { + include: '#jsx-entities', + }, + ], + }, + 'jsx-tag-attributes-illegal': { + name: 'invalid.illegal.attribute.tsx', + match: '\\S+', + }, + }, +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/abyss.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/abyss.ts new file mode 100644 index 0000000..7e70bba --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/abyss.ts @@ -0,0 +1,343 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const abyss: ThemeRegistrationAny = { + type: 'dark', + colors: { + 'activityBar.background': '#051336', + 'badge.background': '#0063a5', + 'button.background': '#2b3c5d', + 'debugExceptionWidget.background': '#051336', + 'debugExceptionWidget.border': '#ab395b', + 'debugToolBar.background': '#051336', + 'diffEditor.insertedTextBackground': '#31958a55', + 'diffEditor.removedTextBackground': '#892f4688', + 'dropdown.background': '#181f2f', + 'editor.background': '#000c18', + 'editor.findMatchHighlightBackground': '#eeeeee44', + 'editor.foreground': '#6688cc', + 'editor.lineHighlightBackground': '#082050', + 'editor.selectionBackground': '#770811', + 'editorCursor.foreground': '#ddbb88', + 'editorGroup.border': '#2b2b4a', + 'editorGroup.dropBackground': '#25375daa', + 'editorGroupHeader.tabsBackground': '#1c1c2a', + 'editorHoverWidget.background': '#000c38', + 'editorHoverWidget.border': '#004c18', + 'editorIndentGuide.activeBackground': '#204972', + 'editorIndentGuide.background': '#002952', + 'editorLineNumber.activeForeground': '#80a2c2', + 'editorLineNumber.foreground': '#406385', + 'editorLink.activeForeground': '#0063a5', + 'editorMarkerNavigation.background': '#060621', + 'editorMarkerNavigationError.background': '#ab395b', + 'editorMarkerNavigationWarning.background': '#5b7e7a', + 'editorWhitespace.foreground': '#103050', + 'editorWidget.background': '#262641', + 'extensionButton.prominentBackground': '#5f8b3b', + 'extensionButton.prominentHoverBackground': '#5f8b3bbb', + focusBorder: '#596f99', + 'input.background': '#181f2f', + 'inputOption.activeBorder': '#1d4a87', + 'inputValidation.errorBackground': '#a22d44', + 'inputValidation.errorBorder': '#ab395b', + 'inputValidation.infoBackground': '#051336', + 'inputValidation.infoBorder': '#384078', + 'inputValidation.warningBackground': '#5b7e7a', + 'inputValidation.warningBorder': '#5b7e7a', + 'list.activeSelectionBackground': '#08286b', + 'list.dropBackground': '#041d52', + 'list.highlightForeground': '#0063a5', + 'list.hoverBackground': '#061940', + 'list.inactiveSelectionBackground': '#152037', + 'minimap.selectionHighlight': '#750000', + 'panel.border': '#2b2b4a', + 'peekView.border': '#2b2b4a', + 'peekViewEditor.background': '#10192c', + 'peekViewEditor.matchHighlightBackground': '#eeeeee33', + 'peekViewResult.background': '#060621', + 'peekViewResult.matchHighlightBackground': '#eeeeee44', + 'peekViewTitle.background': '#10192c', + 'pickerGroup.border': '#596f99', + 'pickerGroup.foreground': '#596f99', + 'ports.iconRunningProcessForeground': '#80a2c2', + 'progressBar.background': '#0063a5', + 'quickInputList.focusBackground': '#08286b', + 'scrollbar.shadow': '#515e91aa', + 'scrollbarSlider.activeBackground': '#3b3f5188', + 'scrollbarSlider.background': '#1f2230aa', + 'scrollbarSlider.hoverBackground': '#3b3f5188', + 'sideBar.background': '#060621', + 'sideBarSectionHeader.background': '#10192c', + 'statusBar.background': '#10192c', + 'statusBar.debuggingBackground': '#10192c', + 'statusBar.noFolderBackground': '#10192c', + 'statusBarItem.prominentBackground': '#0063a5', + 'statusBarItem.prominentHoverBackground': '#0063a5dd', + 'statusBarItem.remoteBackground': '#0063a5', + 'tab.border': '#2b2b4a', + 'tab.inactiveBackground': '#10192c', + 'tab.lastPinnedBorder': '#2b3c5d', + 'terminal.ansiBlack': '#111111', + 'terminal.ansiBlue': '#bbdaff', + 'terminal.ansiBrightBlack': '#333333', + 'terminal.ansiBrightBlue': '#80baff', + 'terminal.ansiBrightCyan': '#78ffff', + 'terminal.ansiBrightGreen': '#b8f171', + 'terminal.ansiBrightMagenta': '#d778ff', + 'terminal.ansiBrightRed': '#ff7882', + 'terminal.ansiBrightWhite': '#ffffff', + 'terminal.ansiBrightYellow': '#ffe580', + 'terminal.ansiCyan': '#99ffff', + 'terminal.ansiGreen': '#d1f1a9', + 'terminal.ansiMagenta': '#ebbbff', + 'terminal.ansiRed': '#ff9da4', + 'terminal.ansiWhite': '#cccccc', + 'terminal.ansiYellow': '#ffeead', + 'titleBar.activeBackground': '#10192c', + }, + tokenColors: [ + { + scope: ['meta.embedded', 'source.groovy.embedded', 'string meta.image.inline.markdown'], + settings: { + foreground: '#6688CC', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#384887', + }, + }, + { + scope: 'string', + settings: { + foreground: '#22AA44', + }, + }, + { + scope: 'constant.numeric', + settings: { + foreground: '#F280D0', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#F280D0', + }, + }, + { + scope: ['constant.character', 'constant.other'], + settings: { + foreground: '#F280D0', + }, + }, + { + scope: 'variable', + settings: { + fontStyle: '', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#225588', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#225588', + fontStyle: '', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#9966B8', + fontStyle: 'italic', + }, + }, + { + scope: ['entity.name.class', 'entity.name.type', 'entity.name.namespace', 'entity.name.scope-resolution'], + settings: { + foreground: '#FFEEBB', + fontStyle: 'underline', + }, + }, + { + scope: 'entity.other.inherited-class', + settings: { + foreground: '#DDBB88', + fontStyle: 'italic underline', + }, + }, + { + scope: 'entity.name.function', + settings: { + foreground: '#DDBB88', + fontStyle: '', + }, + }, + { + scope: 'variable.parameter', + settings: { + foreground: '#2277FF', + fontStyle: 'italic', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#225588', + fontStyle: '', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#DDBB88', + fontStyle: '', + }, + }, + { + scope: 'support.function', + settings: { + foreground: '#9966B8', + fontStyle: '', + }, + }, + { + scope: 'support.constant', + settings: { + foreground: '#9966B8', + fontStyle: '', + }, + }, + { + scope: ['support.type', 'support.class'], + settings: { + foreground: '#9966B8', + fontStyle: 'italic', + }, + }, + { + scope: 'support.other.variable', + settings: { + fontStyle: '', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#A22D44', + fontStyle: '', + }, + }, + { + scope: 'invalid.deprecated', + settings: { + foreground: '#A22D44', + }, + }, + { + scope: ['meta.diff', 'meta.diff.header'], + settings: { + foreground: '#E0EDDD', + fontStyle: 'italic', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#DC322F', + fontStyle: '', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#CB4B16', + fontStyle: '', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#219186', + }, + }, + { + scope: 'markup.quote', + settings: { + foreground: '#22AA44', + }, + }, + { + scope: ['markup.bold', 'markup.italic'], + settings: { + foreground: '#22AA44', + }, + }, + { + scope: 'markup.bold', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#9966B8', + fontStyle: '', + }, + }, + { + scope: ['markup.heading', 'markup.heading.setext'], + settings: { + foreground: '#6688CC', + fontStyle: 'bold', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/dark-hc.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/dark-hc.ts new file mode 100644 index 0000000..c0cd483 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/dark-hc.ts @@ -0,0 +1,462 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const darkHC: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'actionBar.toggledBackground': '#383a49', + 'editor.background': '#000000', + 'editor.foreground': '#ffffff', + 'editor.selectionBackground': '#ffffff', + 'editorIndentGuide.activeBackground1': '#ffffff', + 'editorIndentGuide.background1': '#ffffff', + 'editorWhitespace.foreground': '#7c7c7c', + 'ports.iconRunningProcessForeground': '#ffffff', + 'selection.background': '#008000', + 'sideBarTitle.foreground': '#ffffff', + 'statusBarItem.remoteBackground': '#00000000', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'emphasis', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'strong', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'meta.diff.header', + settings: { + foreground: '#000080', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#7CA668', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: [ + 'constant.numeric', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'constant.regexp', + settings: { + foreground: '#B46695', + }, + }, + { + scope: 'constant.character', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'entity.name.tag.css', + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: [ + 'entity.other.attribute-name.class.css', + 'entity.other.attribute-name.class.mixin.css', + 'entity.other.attribute-name.id.css', + 'entity.other.attribute-name.parent-selector.css', + 'entity.other.attribute-name.pseudo-class.css', + 'entity.other.attribute-name.pseudo-element.css', + 'source.css.less entity.other.attribute-name.id', + 'entity.other.attribute-name.scss', + ], + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.bold', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#6796E6', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['punctuation.definition.tag'], + settings: { + foreground: '#808080', + }, + }, + { + scope: 'meta.preprocessor', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'meta.preprocessor.string', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'meta.preprocessor.numeric', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'meta.structure.dictionary.key.python', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'storage.modifier', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'string', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.tag', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.value', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#D16969', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + ], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: [ + 'support.type.vendored.property-name', + 'support.type.property-name', + 'variable.css', + 'variable.scss', + 'variable.other.less', + 'source.coffee.embedded', + ], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.control', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: [ + 'keyword.operator.new', + 'keyword.operator.expression', + 'keyword.operator.cast', + 'keyword.operator.sizeof', + 'keyword.operator.logical.python', + ], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'support.function.git-rebase', + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'constant.sha.git-rebase', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: ['storage.modifier.import.java', 'variable.language.wildcard.java', 'storage.modifier.package.java'], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'variable.language.this', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: [ + 'entity.name.function', + 'support.function', + 'support.constant.handlebars', + 'source.powershell variable.other.member', + ], + settings: { + foreground: '#DCDCAA', + }, + }, + { + scope: [ + 'support.class', + 'support.type', + 'entity.name.type', + 'entity.name.namespace', + 'entity.name.scope-resolution', + 'entity.name.class', + 'storage.type.cs', + 'storage.type.generic.cs', + 'storage.type.modifier.cs', + 'storage.type.variable.cs', + 'storage.type.annotation.java', + 'storage.type.generic.java', + 'storage.type.java', + 'storage.type.object.array.java', + 'storage.type.primitive.array.java', + 'storage.type.primitive.java', + 'storage.type.token.java', + 'storage.type.groovy', + 'storage.type.annotation.groovy', + 'storage.type.parameters.groovy', + 'storage.type.generic.groovy', + 'storage.type.object.array.groovy', + 'storage.type.primitive.array.groovy', + 'storage.type.primitive.groovy', + ], + settings: { + foreground: '#4EC9B0', + }, + }, + { + scope: [ + 'meta.type.cast.expr', + 'meta.type.new.expr', + 'support.constant.math', + 'support.constant.dom', + 'support.constant.json', + 'entity.other.inherited-class', + ], + settings: { + foreground: '#4EC9B0', + }, + }, + { + scope: [ + 'keyword.control', + 'source.cpp keyword.operator.new', + 'source.cpp keyword.operator.delete', + 'keyword.other.using', + 'keyword.other.directive.using', + 'keyword.other.operator', + ], + settings: { + foreground: '#C586C0', + }, + }, + { + scope: ['variable', 'meta.definition.variable.name', 'support.variable'], + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: ['meta.object-literal.key'], + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'meta.resultLinePrefix.contextLinePrefix.search', + settings: { + foreground: '#CBEDCB', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#008000', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#FF0000', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/dark-modern.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/dark-modern.ts new file mode 100644 index 0000000..1fb2392 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/dark-modern.ts @@ -0,0 +1,692 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const darkModern: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'actionBar.toggledBackground': '#383a49', + 'activityBar.activeBorder': '#0078d4', + 'activityBar.background': '#181818', + 'activityBar.border': '#2b2b2b', + 'activityBar.foreground': '#d7d7d7', + 'activityBar.inactiveForeground': '#868686', + 'activityBarBadge.background': '#0078d4', + 'activityBarBadge.foreground': '#ffffff', + 'badge.background': '#616161', + 'badge.foreground': '#f8f8f8', + 'button.background': '#0078d4', + 'button.border': '#ffffff12', + 'button.foreground': '#ffffff', + 'button.hoverBackground': '#026ec1', + 'button.secondaryBackground': '#313131', + 'button.secondaryForeground': '#cccccc', + 'button.secondaryHoverBackground': '#3c3c3c', + 'chat.slashCommandBackground': '#34414b', + 'chat.slashCommandForeground': '#40a6ff', + 'checkbox.background': '#313131', + 'checkbox.border': '#3c3c3c', + 'debugToolBar.background': '#181818', + descriptionForeground: '#9d9d9d', + 'dropdown.background': '#313131', + 'dropdown.border': '#3c3c3c', + 'dropdown.foreground': '#cccccc', + 'dropdown.listBackground': '#1f1f1f', + 'editor.background': '#1f1f1f', + 'editor.findMatchBackground': '#9e6a03', + 'editor.foreground': '#cccccc', + 'editor.inactiveSelectionBackground': '#3a3d41', + 'editor.selectionHighlightBackground': '#add6ff26', + 'editorGroup.border': '#ffffff17', + 'editorGroupHeader.tabsBackground': '#181818', + 'editorGroupHeader.tabsBorder': '#2b2b2b', + 'editorGutter.addedBackground': '#2ea043', + 'editorGutter.deletedBackground': '#f85149', + 'editorGutter.modifiedBackground': '#0078d4', + 'editorIndentGuide.activeBackground1': '#707070', + 'editorIndentGuide.background1': '#404040', + 'editorLineNumber.activeForeground': '#cccccc', + 'editorLineNumber.foreground': '#6e7681', + 'editorOverviewRuler.border': '#010409', + 'editorWidget.background': '#202020', + errorForeground: '#f85149', + focusBorder: '#0078d4', + foreground: '#cccccc', + 'icon.foreground': '#cccccc', + 'input.background': '#313131', + 'input.border': '#3c3c3c', + 'input.foreground': '#cccccc', + 'input.placeholderForeground': '#818181', + 'inputOption.activeBackground': '#2489db82', + 'inputOption.activeBorder': '#2488db', + 'keybindingLabel.foreground': '#cccccc', + 'list.activeSelectionIconForeground': '#ffffff', + 'list.dropBackground': '#383b3d', + 'menu.background': '#1f1f1f', + 'menu.border': '#454545', + 'menu.foreground': '#cccccc', + 'menu.separatorBackground': '#454545', + 'notificationCenterHeader.background': '#1f1f1f', + 'notificationCenterHeader.foreground': '#cccccc', + 'notifications.background': '#1f1f1f', + 'notifications.border': '#2b2b2b', + 'notifications.foreground': '#cccccc', + 'panel.background': '#181818', + 'panel.border': '#2b2b2b', + 'panelInput.border': '#2b2b2b', + 'panelTitle.activeBorder': '#0078d4', + 'panelTitle.activeForeground': '#cccccc', + 'panelTitle.inactiveForeground': '#9d9d9d', + 'peekViewEditor.background': '#1f1f1f', + 'peekViewEditor.matchHighlightBackground': '#bb800966', + 'peekViewResult.background': '#1f1f1f', + 'peekViewResult.matchHighlightBackground': '#bb800966', + 'pickerGroup.border': '#3c3c3c', + 'ports.iconRunningProcessForeground': '#369432', + 'progressBar.background': '#0078d4', + 'quickInput.background': '#222222', + 'quickInput.foreground': '#cccccc', + 'settings.dropdownBackground': '#313131', + 'settings.dropdownBorder': '#3c3c3c', + 'settings.headerForeground': '#ffffff', + 'settings.modifiedItemIndicator': '#bb800966', + 'sideBar.background': '#181818', + 'sideBar.border': '#2b2b2b', + 'sideBar.foreground': '#cccccc', + 'sideBarSectionHeader.background': '#181818', + 'sideBarSectionHeader.border': '#2b2b2b', + 'sideBarSectionHeader.foreground': '#cccccc', + 'sideBarTitle.foreground': '#cccccc', + 'statusBar.background': '#181818', + 'statusBar.border': '#2b2b2b', + 'statusBar.debuggingBackground': '#0078d4', + 'statusBar.debuggingForeground': '#ffffff', + 'statusBar.focusBorder': '#0078d4', + 'statusBar.foreground': '#cccccc', + 'statusBar.noFolderBackground': '#1f1f1f', + 'statusBarItem.focusBorder': '#0078d4', + 'statusBarItem.prominentBackground': '#6e768166', + 'statusBarItem.remoteBackground': '#0078d4', + 'statusBarItem.remoteForeground': '#ffffff', + 'tab.activeBackground': '#1f1f1f', + 'tab.activeBorder': '#1f1f1f', + 'tab.activeBorderTop': '#0078d4', + 'tab.activeForeground': '#ffffff', + 'tab.border': '#2b2b2b', + 'tab.hoverBackground': '#1f1f1f', + 'tab.inactiveBackground': '#181818', + 'tab.inactiveForeground': '#9d9d9d', + 'tab.lastPinnedBorder': '#cccccc33', + 'tab.unfocusedActiveBorder': '#1f1f1f', + 'tab.unfocusedActiveBorderTop': '#2b2b2b', + 'tab.unfocusedHoverBackground': '#1f1f1f', + 'terminal.foreground': '#cccccc', + 'terminal.inactiveSelectionBackground': '#3a3d41', + 'terminal.tab.activeBorder': '#0078d4', + 'textBlockQuote.background': '#2b2b2b', + 'textBlockQuote.border': '#616161', + 'textCodeBlock.background': '#2b2b2b', + 'textLink.activeForeground': '#4daafc', + 'textLink.foreground': '#4daafc', + 'textPreformat.background': '#3c3c3c', + 'textPreformat.foreground': '#d0d0d0', + 'textSeparator.foreground': '#21262d', + 'titleBar.activeBackground': '#181818', + 'titleBar.activeForeground': '#cccccc', + 'titleBar.border': '#2b2b2b', + 'titleBar.inactiveBackground': '#1f1f1f', + 'titleBar.inactiveForeground': '#9d9d9d', + 'welcomePage.progress.foreground': '#0078d4', + 'welcomePage.tileBackground': '#2b2b2b', + 'widget.border': '#313131', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'emphasis', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'strong', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'header', + settings: { + foreground: '#000080', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#6A9955', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: [ + 'constant.numeric', + 'variable.other.enummember', + 'keyword.operator.plus.exponent', + 'keyword.operator.minus.exponent', + ], + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'constant.regexp', + settings: { + foreground: '#646695', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'entity.name.tag.css', + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: [ + 'entity.other.attribute-name.class.css', + 'entity.other.attribute-name.class.mixin.css', + 'entity.other.attribute-name.id.css', + 'entity.other.attribute-name.parent-selector.css', + 'entity.other.attribute-name.pseudo-class.css', + 'entity.other.attribute-name.pseudo-element.css', + 'source.css.less entity.other.attribute-name.id', + 'entity.other.attribute-name.scss', + ], + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.bold', + settings: { + foreground: '#569CD6', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#569CD6', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'punctuation.definition.quote.begin.markdown', + settings: { + foreground: '#6A9955', + }, + }, + { + scope: 'punctuation.definition.list.begin.markdown', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'punctuation.definition.tag', + settings: { + foreground: '#808080', + }, + }, + { + scope: ['meta.preprocessor', 'entity.name.function.preprocessor'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'meta.preprocessor.string', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'meta.preprocessor.numeric', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'meta.structure.dictionary.key.python', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'meta.diff.header', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['storage.modifier', 'keyword.operator.noexcept'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['string', 'meta.embedded.assembly'], + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.tag', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.value', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#D16969', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + ], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: [ + 'support.type.vendored.property-name', + 'support.type.property-name', + 'variable.css', + 'variable.scss', + 'variable.other.less', + 'source.coffee.embedded', + ], + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.control', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: [ + 'keyword.operator.new', + 'keyword.operator.expression', + 'keyword.operator.cast', + 'keyword.operator.sizeof', + 'keyword.operator.alignof', + 'keyword.operator.typeid', + 'keyword.operator.alignas', + 'keyword.operator.instanceof', + 'keyword.operator.logical.python', + 'keyword.operator.wordlike', + ], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: ['punctuation.section.embedded.begin.php', 'punctuation.section.embedded.end.php'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'support.function.git-rebase', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'constant.sha.git-rebase', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: ['storage.modifier.import.java', 'variable.language.wildcard.java', 'storage.modifier.package.java'], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'variable.language', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: [ + 'entity.name.function', + 'support.function', + 'support.constant.handlebars', + 'source.powershell variable.other.member', + 'entity.name.operator.custom-literal', + ], + settings: { + foreground: '#DCDCAA', + }, + }, + { + scope: [ + 'support.class', + 'support.type', + 'entity.name.type', + 'entity.name.namespace', + 'entity.other.attribute', + 'entity.name.scope-resolution', + 'entity.name.class', + 'storage.type.numeric.go', + 'storage.type.byte.go', + 'storage.type.boolean.go', + 'storage.type.string.go', + 'storage.type.uintptr.go', + 'storage.type.error.go', + 'storage.type.rune.go', + 'storage.type.cs', + 'storage.type.generic.cs', + 'storage.type.modifier.cs', + 'storage.type.variable.cs', + 'storage.type.annotation.java', + 'storage.type.generic.java', + 'storage.type.java', + 'storage.type.object.array.java', + 'storage.type.primitive.array.java', + 'storage.type.primitive.java', + 'storage.type.token.java', + 'storage.type.groovy', + 'storage.type.annotation.groovy', + 'storage.type.parameters.groovy', + 'storage.type.generic.groovy', + 'storage.type.object.array.groovy', + 'storage.type.primitive.array.groovy', + 'storage.type.primitive.groovy', + ], + settings: { + foreground: '#4EC9B0', + }, + }, + { + scope: [ + 'meta.type.cast.expr', + 'meta.type.new.expr', + 'support.constant.math', + 'support.constant.dom', + 'support.constant.json', + 'entity.other.inherited-class', + ], + settings: { + foreground: '#4EC9B0', + }, + }, + { + scope: [ + 'keyword.control', + 'source.cpp keyword.operator.new', + 'keyword.operator.delete', + 'keyword.other.using', + 'keyword.other.directive.using', + 'keyword.other.operator', + 'entity.name.operator', + ], + settings: { + foreground: '#C586C0', + }, + }, + { + scope: [ + 'variable', + 'meta.definition.variable.name', + 'support.variable', + 'entity.name.variable', + 'constant.other.placeholder', + ], + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: ['variable.other.constant', 'variable.other.enummember'], + settings: { + foreground: '#4FC1FF', + }, + }, + { + scope: ['meta.object-literal.key'], + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#CE9178', + }, + }, + { + scope: [ + 'punctuation.definition.group.regexp', + 'punctuation.definition.group.assertion.regexp', + 'punctuation.definition.character-class.regexp', + 'punctuation.character.set.begin.regexp', + 'punctuation.character.set.end.regexp', + 'keyword.operator.negation.regexp', + 'support.other.parenthesis.regexp', + ], + settings: { + foreground: '#CE9178', + }, + }, + { + scope: [ + 'constant.character.character-class.regexp', + 'constant.other.character-class.set.regexp', + 'constant.other.character-class.regexp', + 'constant.character.set.regexp', + ], + settings: { + foreground: '#D16969', + }, + }, + { + scope: ['keyword.operator.or.regexp', 'keyword.control.anchor.regexp'], + settings: { + foreground: '#DCDCAA', + }, + }, + { + scope: 'keyword.operator.quantifier.regexp', + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: ['constant.character', 'constant.other.option'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'constant.character.escape', + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'entity.name.label', + settings: { + foreground: '#C8C8C8', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/index.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/index.ts new file mode 100644 index 0000000..21d9e04 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/index.ts @@ -0,0 +1,21 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export { default as darkPlus } from 'shiki/themes/dark-plus.mjs'; +export { default as lightPlus } from 'shiki/themes/light-plus.mjs'; +export { default as monokai } from 'shiki/themes/monokai.mjs'; +export { default as solarizedDark } from 'shiki/themes/solarized-dark.mjs'; +export { default as solarizedLight } from 'shiki/themes/solarized-light.mjs'; +export { abyss } from './abyss'; +export { darkHC } from './dark-hc'; +export { darkModern } from './dark-modern'; +export { kimbieDark } from './kimbie-dark'; +export { lightHC } from './light-hc'; +export { lightModern } from './light-modern'; +export { monokaiDim } from './monokai-dim'; +export { quietLight } from './quiet-light'; +export { red } from './red'; +export { tomorrowNightBlue } from './tomorrow-night-blue'; +export { vsDark } from './vs-dark'; +export { vsLight } from './vs-light'; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/kimbie-dark.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/kimbie-dark.ts new file mode 100644 index 0000000..d9289f6 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/kimbie-dark.ts @@ -0,0 +1,374 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const kimbieDark: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'activityBar.background': '#221a0f', + 'activityBar.foreground': '#d3af86', + 'badge.background': '#7f5d38', + 'button.background': '#6e583b', + 'dropdown.background': '#51412c', + 'editor.background': '#221a0f', + 'editor.foreground': '#d3af86', + 'editor.lineHighlightBackground': '#5e452b', + 'editor.selectionBackground': '#84613daa', + 'editorCursor.foreground': '#d3af86', + 'editorGroupHeader.tabsBackground': '#131510', + 'editorHoverWidget.background': '#221a14', + 'editorLineNumber.activeForeground': '#adadad', + 'editorWhitespace.foreground': '#a57a4c', + 'editorWidget.background': '#131510', + focusBorder: '#a57a4c', + 'input.background': '#51412c', + 'inputOption.activeBorder': '#a57a4c', + 'inputValidation.errorBackground': '#5f0d0d', + 'inputValidation.errorBorder': '#9d2f23', + 'inputValidation.infoBackground': '#2b2a42', + 'inputValidation.infoBorder': '#1b60a5', + 'inputValidation.warningBackground': '#51412c', + 'list.activeSelectionBackground': '#7c5021', + 'list.highlightForeground': '#e3b583', + 'list.hoverBackground': '#7c502166', + 'list.inactiveSelectionBackground': '#645342', + 'menu.background': '#362712', + 'menu.foreground': '#cccccc', + 'minimap.selectionHighlight': '#84613daa', + 'peekView.border': '#5e452b', + 'peekViewEditor.background': '#221a14', + 'peekViewEditor.matchHighlightBackground': '#84613daa', + 'peekViewResult.background': '#362712', + 'peekViewTitle.background': '#362712', + 'pickerGroup.border': '#e3b583', + 'pickerGroup.foreground': '#e3b583', + 'ports.iconRunningProcessForeground': '#369432', + 'progressBar.background': '#7f5d38', + 'quickInputList.focusBackground': '#7c5021aa', + 'selection.background': '#84613daa', + 'sideBar.background': '#362712', + 'statusBar.background': '#423523', + 'statusBar.debuggingBackground': '#423523', + 'statusBar.noFolderBackground': '#423523', + 'statusBarItem.remoteBackground': '#6e583b', + 'tab.inactiveBackground': '#131510', + 'tab.lastPinnedBorder': '#51412c', + 'titleBar.activeBackground': '#423523', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#D3AF86', + }, + }, + { + scope: 'variable.parameter.function', + settings: { + foreground: '#D3AF86', + }, + }, + { + scope: ['comment', 'punctuation.definition.comment'], + settings: { + foreground: '#A57A4C', + }, + }, + { + scope: [ + 'punctuation.definition.string', + 'punctuation.definition.variable', + 'punctuation.definition.string', + 'punctuation.definition.parameters', + 'punctuation.definition.string', + 'punctuation.definition.array', + ], + settings: { + foreground: '#D3AF86', + }, + }, + { + scope: 'none', + settings: { + foreground: '#D3AF86', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#D3AF86', + }, + }, + { + scope: [ + 'keyword', + 'keyword.control', + 'keyword.operator.new.cpp', + 'keyword.operator.delete.cpp', + 'keyword.other.using', + 'keyword.other.directive.using', + 'keyword.other.operator', + ], + settings: { + foreground: '#98676A', + }, + }, + { + scope: 'variable', + settings: { + foreground: '#DC3958', + }, + }, + { + scope: ['entity.name.function', 'meta.require', 'support.function.any-method'], + settings: { + foreground: '#8AB1B0', + }, + }, + { + scope: [ + 'support.class', + 'entity.name.class', + 'entity.name.type', + 'entity.name.namespace', + 'entity.name.scope-resolution', + ], + settings: { + foreground: '#F06431', + }, + }, + { + scope: 'keyword.other.special-method', + settings: { + foreground: '#8AB1B0', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#98676A', + }, + }, + { + scope: 'support.function', + settings: { + foreground: '#7E602C', + }, + }, + { + scope: ['string', 'constant.other.symbol', 'entity.other.inherited-class'], + settings: { + foreground: '#889B4A', + }, + }, + { + scope: 'constant.numeric', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: 'none', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: 'none', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: 'constant', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#DC3958', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: ['entity.other.attribute-name.id', 'punctuation.definition.entity'], + settings: { + foreground: '#8AB1B0', + }, + }, + { + scope: 'meta.selector', + settings: { + foreground: '#98676A', + }, + }, + { + scope: 'none', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: ['markup.heading', 'markup.heading.setext', 'punctuation.definition.heading', 'entity.name.section'], + settings: { + foreground: '#8AB1B0', + fontStyle: 'bold', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: ['markup.bold', 'punctuation.definition.bold'], + settings: { + foreground: '#F06431', + fontStyle: 'bold', + }, + }, + { + scope: ['markup.italic', 'punctuation.definition.italic'], + settings: { + foreground: '#98676A', + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#889B4A', + }, + }, + { + scope: 'string.other.link', + settings: { + foreground: '#DC3958', + }, + }, + { + scope: 'meta.link', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: 'markup.list', + settings: { + foreground: '#DC3958', + }, + }, + { + scope: 'markup.quote', + settings: { + foreground: '#F79A32', + }, + }, + { + scope: 'meta.separator', + settings: { + foreground: '#D3AF86', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#889B4A', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#DC3958', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#98676A', + }, + }, + { + scope: 'constant.other.color', + settings: { + foreground: '#7E602C', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#7E602C', + }, + }, + { + scope: 'constant.character.escape', + settings: { + foreground: '#7E602C', + }, + }, + { + scope: ['punctuation.section.embedded', 'variable.interpolation'], + settings: { + foreground: '#088649', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#DC3958', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/light-hc.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/light-hc.ts new file mode 100644 index 0000000..390a2c0 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/light-hc.ts @@ -0,0 +1,572 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const lightHC: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'light', + colors: { + 'actionBar.toggledBackground': '#dddddd', + }, + tokenColors: [ + { + scope: ['meta.embedded', 'source.groovy.embedded', 'variable.legacy.builtin.python'], + settings: { + foreground: '#292929', + }, + }, + { + scope: 'emphasis', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'strong', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'meta.diff.header', + settings: { + foreground: '#062F4A', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#515151', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: [ + 'constant.numeric', + 'variable.other.enummember', + 'keyword.operator.plus.exponent', + 'keyword.operator.minus.exponent', + ], + settings: { + foreground: '#096D48', + }, + }, + { + scope: 'constant.regexp', + settings: { + foreground: '#811F3F', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'entity.name.selector', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#264F78', + }, + }, + { + scope: [ + 'entity.other.attribute-name.class.css', + 'entity.other.attribute-name.class.mixin.css', + 'entity.other.attribute-name.id.css', + 'entity.other.attribute-name.parent-selector.css', + 'entity.other.attribute-name.pseudo-class.css', + 'entity.other.attribute-name.pseudo-element.css', + 'source.css.less entity.other.attribute-name.id', + 'entity.other.attribute-name.scss', + ], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#B5200D', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.bold', + settings: { + foreground: '#000080', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#0F4A85', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#096D48', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#5A5A5A', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: ['punctuation.definition.quote.begin.markdown', 'punctuation.definition.list.begin.markdown'], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'punctuation.definition.tag', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: ['meta.preprocessor', 'entity.name.function.preprocessor'], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'meta.preprocessor.string', + settings: { + foreground: '#B5200D', + }, + }, + { + scope: 'meta.preprocessor.numeric', + settings: { + foreground: '#096D48', + }, + }, + { + scope: 'meta.structure.dictionary.key.python', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: ['storage.modifier', 'keyword.operator.noexcept'], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: ['string', 'meta.embedded.assembly'], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: [ + 'string.comment.buffered.block.pug', + 'string.quoted.pug', + 'string.interpolated.pug', + 'string.unquoted.plain.in.yaml', + 'string.unquoted.plain.out.yaml', + 'string.unquoted.block.yaml', + 'string.quoted.single.yaml', + 'string.quoted.double.xml', + 'string.quoted.single.xml', + 'string.unquoted.cdata.xml', + 'string.quoted.double.html', + 'string.quoted.single.html', + 'string.unquoted.html', + 'string.quoted.single.handlebars', + 'string.quoted.double.handlebars', + ], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#811F3F', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + ], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#000000', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: [ + 'support.type.vendored.property-name', + 'support.type.property-name', + 'variable.css', + 'variable.scss', + 'variable.other.less', + 'source.coffee.embedded', + ], + settings: { + foreground: '#264F78', + }, + }, + { + scope: ['support.type.property-name.json'], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'keyword.control', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#000000', + }, + }, + { + scope: [ + 'keyword.operator.new', + 'keyword.operator.expression', + 'keyword.operator.cast', + 'keyword.operator.sizeof', + 'keyword.operator.alignof', + 'keyword.operator.typeid', + 'keyword.operator.alignas', + 'keyword.operator.instanceof', + 'keyword.operator.logical.python', + 'keyword.operator.wordlike', + ], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#096D48', + }, + }, + { + scope: ['punctuation.section.embedded.begin.php', 'punctuation.section.embedded.end.php'], + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'support.function.git-rebase', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'constant.sha.git-rebase', + settings: { + foreground: '#096D48', + }, + }, + { + scope: ['storage.modifier.import.java', 'variable.language.wildcard.java', 'storage.modifier.package.java'], + settings: { + foreground: '#000000', + }, + }, + { + scope: 'variable.language', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: [ + 'entity.name.function', + 'support.function', + 'support.constant.handlebars', + 'source.powershell variable.other.member', + 'entity.name.operator.custom-literal', + ], + settings: { + foreground: '#5E2CBC', + }, + }, + { + scope: [ + 'support.class', + 'support.type', + 'entity.name.type', + 'entity.name.namespace', + 'entity.other.attribute', + 'entity.name.scope-resolution', + 'entity.name.class', + 'storage.type.numeric.go', + 'storage.type.byte.go', + 'storage.type.boolean.go', + 'storage.type.string.go', + 'storage.type.uintptr.go', + 'storage.type.error.go', + 'storage.type.rune.go', + 'storage.type.cs', + 'storage.type.generic.cs', + 'storage.type.modifier.cs', + 'storage.type.variable.cs', + 'storage.type.annotation.java', + 'storage.type.generic.java', + 'storage.type.java', + 'storage.type.object.array.java', + 'storage.type.primitive.array.java', + 'storage.type.primitive.java', + 'storage.type.token.java', + 'storage.type.groovy', + 'storage.type.annotation.groovy', + 'storage.type.parameters.groovy', + 'storage.type.generic.groovy', + 'storage.type.object.array.groovy', + 'storage.type.primitive.array.groovy', + 'storage.type.primitive.groovy', + ], + settings: { + foreground: '#185E73', + }, + }, + { + scope: [ + 'meta.type.cast.expr', + 'meta.type.new.expr', + 'support.constant.math', + 'support.constant.dom', + 'support.constant.json', + 'entity.other.inherited-class', + ], + settings: { + foreground: '#185E73', + }, + }, + { + scope: [ + 'keyword.control', + 'source.cpp keyword.operator.new', + 'source.cpp keyword.operator.delete', + 'keyword.other.using', + 'keyword.other.directive.using', + 'keyword.other.operator', + 'entity.name.operator', + ], + settings: { + foreground: '#B5200D', + }, + }, + { + scope: [ + 'variable', + 'meta.definition.variable.name', + 'support.variable', + 'entity.name.variable', + 'constant.other.placeholder', + ], + settings: { + foreground: '#001080', + }, + }, + { + scope: ['variable.other.constant', 'variable.other.enummember'], + settings: { + foreground: '#02715D', + }, + }, + { + scope: ['meta.object-literal.key'], + settings: { + foreground: '#001080', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: [ + 'punctuation.definition.group.regexp', + 'punctuation.definition.group.assertion.regexp', + 'punctuation.definition.character-class.regexp', + 'punctuation.character.set.begin.regexp', + 'punctuation.character.set.end.regexp', + 'keyword.operator.negation.regexp', + 'support.other.parenthesis.regexp', + ], + settings: { + foreground: '#D16969', + }, + }, + { + scope: [ + 'constant.character.character-class.regexp', + 'constant.other.character-class.set.regexp', + 'constant.other.character-class.regexp', + 'constant.character.set.regexp', + ], + settings: { + foreground: '#811F3F', + }, + }, + { + scope: 'keyword.operator.quantifier.regexp', + settings: { + foreground: '#000000', + }, + }, + { + scope: ['keyword.operator.or.regexp', 'keyword.control.anchor.regexp'], + settings: { + foreground: '#EE0000', + }, + }, + { + scope: 'constant.character', + settings: { + foreground: '#0F4A85', + }, + }, + { + scope: 'constant.character.escape', + settings: { + foreground: '#EE0000', + }, + }, + { + scope: 'entity.name.label', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#316BCD', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#800080', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#000000', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/light-modern.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/light-modern.ts new file mode 100644 index 0000000..2a5a90a --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/light-modern.ts @@ -0,0 +1,716 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const lightModern: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'light', + colors: { + 'actionBar.toggledBackground': '#dddddd', + 'activityBar.activeBorder': '#005fb8', + 'activityBar.background': '#f8f8f8', + 'activityBar.border': '#e5e5e5', + 'activityBar.foreground': '#1f1f1f', + 'activityBar.inactiveForeground': '#616161', + 'activityBarBadge.background': '#005fb8', + 'activityBarBadge.foreground': '#ffffff', + 'badge.background': '#cccccc', + 'badge.foreground': '#3b3b3b', + 'button.background': '#005fb8', + 'button.border': '#0000001a', + 'button.foreground': '#ffffff', + 'button.hoverBackground': '#0258a8', + 'button.secondaryBackground': '#e5e5e5', + 'button.secondaryForeground': '#3b3b3b', + 'button.secondaryHoverBackground': '#cccccc', + 'chat.slashCommandBackground': '#d2ecff', + 'chat.slashCommandForeground': '#306ca2', + 'checkbox.background': '#f8f8f8', + 'checkbox.border': '#cecece', + descriptionForeground: '#3b3b3b', + 'dropdown.background': '#ffffff', + 'dropdown.border': '#cecece', + 'dropdown.foreground': '#3b3b3b', + 'dropdown.listBackground': '#ffffff', + 'editor.background': '#ffffff', + 'editor.foreground': '#3b3b3b', + 'editor.inactiveSelectionBackground': '#e5ebf1', + 'editor.selectionHighlightBackground': '#add6ff80', + 'editorGroup.border': '#e5e5e5', + 'editorGroupHeader.tabsBackground': '#f8f8f8', + 'editorGroupHeader.tabsBorder': '#e5e5e5', + 'editorGutter.addedBackground': '#2ea043', + 'editorGutter.deletedBackground': '#f85149', + 'editorGutter.modifiedBackground': '#005fb8', + 'editorIndentGuide.activeBackground1': '#939393', + 'editorIndentGuide.background1': '#d3d3d3', + 'editorLineNumber.activeForeground': '#171184', + 'editorLineNumber.foreground': '#6e7681', + 'editorOverviewRuler.border': '#e5e5e5', + 'editorSuggestWidget.background': '#f8f8f8', + 'editorWidget.background': '#f8f8f8', + errorForeground: '#f85149', + focusBorder: '#005fb8', + foreground: '#3b3b3b', + 'icon.foreground': '#3b3b3b', + 'input.background': '#ffffff', + 'input.border': '#cecece', + 'input.foreground': '#3b3b3b', + 'input.placeholderForeground': '#868686', + 'inputOption.activeBackground': '#bed6ed', + 'inputOption.activeBorder': '#005fb8', + 'inputOption.activeForeground': '#000000', + 'keybindingLabel.foreground': '#3b3b3b', + 'list.activeSelectionBackground': '#e8e8e8', + 'list.activeSelectionForeground': '#000000', + 'list.activeSelectionIconForeground': '#000000', + 'list.focusAndSelectionOutline': '#005fb8', + 'list.hoverBackground': '#f2f2f2', + 'menu.border': '#cecece', + 'notebook.cellBorderColor': '#e5e5e5', + 'notebook.selectedCellBackground': '#c8ddf150', + 'notificationCenterHeader.background': '#ffffff', + 'notificationCenterHeader.foreground': '#3b3b3b', + 'notifications.background': '#ffffff', + 'notifications.border': '#e5e5e5', + 'notifications.foreground': '#3b3b3b', + 'panel.background': '#f8f8f8', + 'panel.border': '#e5e5e5', + 'panelInput.border': '#e5e5e5', + 'panelTitle.activeBorder': '#005fb8', + 'panelTitle.activeForeground': '#3b3b3b', + 'panelTitle.inactiveForeground': '#3b3b3b', + 'peekViewEditor.matchHighlightBackground': '#bb800966', + 'peekViewResult.background': '#ffffff', + 'peekViewResult.matchHighlightBackground': '#bb800966', + 'pickerGroup.border': '#e5e5e5', + 'pickerGroup.foreground': '#8b949e', + 'ports.iconRunningProcessForeground': '#369432', + 'progressBar.background': '#005fb8', + 'quickInput.background': '#f8f8f8', + 'quickInput.foreground': '#3b3b3b', + 'searchEditor.textInputBorder': '#cecece', + 'settings.dropdownBackground': '#ffffff', + 'settings.dropdownBorder': '#cecece', + 'settings.headerForeground': '#1f1f1f', + 'settings.modifiedItemIndicator': '#bb800966', + 'settings.numberInputBorder': '#cecece', + 'settings.textInputBorder': '#cecece', + 'sideBar.background': '#f8f8f8', + 'sideBar.border': '#e5e5e5', + 'sideBar.foreground': '#3b3b3b', + 'sideBarSectionHeader.background': '#f8f8f8', + 'sideBarSectionHeader.border': '#e5e5e5', + 'sideBarSectionHeader.foreground': '#3b3b3b', + 'sideBarTitle.foreground': '#3b3b3b', + 'statusBar.background': '#f8f8f8', + 'statusBar.border': '#e5e5e5', + 'statusBar.debuggingBackground': '#fd716c', + 'statusBar.debuggingForeground': '#000000', + 'statusBar.focusBorder': '#005fb8', + 'statusBar.foreground': '#3b3b3b', + 'statusBar.noFolderBackground': '#f8f8f8', + 'statusBarItem.errorBackground': '#c72e0f', + 'statusBarItem.focusBorder': '#005fb8', + 'statusBarItem.prominentBackground': '#6e768166', + 'statusBarItem.remoteBackground': '#005fb8', + 'statusBarItem.remoteForeground': '#ffffff', + 'tab.activeBackground': '#ffffff', + 'tab.activeBorder': '#f8f8f8', + 'tab.activeBorderTop': '#005fb8', + 'tab.activeForeground': '#3b3b3b', + 'tab.border': '#e5e5e5', + 'tab.hoverBackground': '#ffffff', + 'tab.inactiveBackground': '#f8f8f8', + 'tab.inactiveForeground': '#868686', + 'tab.lastPinnedBorder': '#d4d4d4', + 'tab.unfocusedActiveBorder': '#f8f8f8', + 'tab.unfocusedActiveBorderTop': '#e5e5e5', + 'tab.unfocusedHoverBackground': '#f8f8f8', + 'terminal.foreground': '#3b3b3b', + 'terminal.inactiveSelectionBackground': '#e5ebf1', + 'terminal.tab.activeBorder': '#005fb8', + 'terminalCursor.foreground': '#005fb8', + 'textBlockQuote.background': '#f8f8f8', + 'textBlockQuote.border': '#e5e5e5', + 'textCodeBlock.background': '#f8f8f8', + 'textLink.activeForeground': '#005fb8', + 'textLink.foreground': '#005fb8', + 'textPreformat.background': '#0000001f', + 'textPreformat.foreground': '#3b3b3b', + 'textSeparator.foreground': '#21262d', + 'titleBar.activeBackground': '#f8f8f8', + 'titleBar.activeForeground': '#1e1e1e', + 'titleBar.border': '#e5e5e5', + 'titleBar.inactiveBackground': '#f8f8f8', + 'titleBar.inactiveForeground': '#8b949e', + 'welcomePage.tileBackground': '#f3f3f3', + 'widget.border': '#e5e5e5', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#000000', + }, + }, + { + scope: 'emphasis', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'strong', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'meta.diff.header', + settings: { + foreground: '#000080', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#008000', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: [ + 'constant.numeric', + 'variable.other.enummember', + 'keyword.operator.plus.exponent', + 'keyword.operator.minus.exponent', + ], + settings: { + foreground: '#098658', + }, + }, + { + scope: 'constant.regexp', + settings: { + foreground: '#811F3F', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#800000', + }, + }, + { + scope: 'entity.name.selector', + settings: { + foreground: '#800000', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#E50000', + }, + }, + { + scope: [ + 'entity.other.attribute-name.class.css', + 'entity.other.attribute-name.class.mixin.css', + 'entity.other.attribute-name.id.css', + 'entity.other.attribute-name.parent-selector.css', + 'entity.other.attribute-name.pseudo-class.css', + 'entity.other.attribute-name.pseudo-element.css', + 'source.css.less entity.other.attribute-name.id', + 'entity.other.attribute-name.scss', + ], + settings: { + foreground: '#800000', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.bold', + settings: { + foreground: '#000080', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#800000', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#098658', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#A31515', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: ['punctuation.definition.quote.begin.markdown', 'punctuation.definition.list.begin.markdown'], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#800000', + }, + }, + { + scope: 'punctuation.definition.tag', + settings: { + foreground: '#800000', + }, + }, + { + scope: ['meta.preprocessor', 'entity.name.function.preprocessor'], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'meta.preprocessor.string', + settings: { + foreground: '#A31515', + }, + }, + { + scope: 'meta.preprocessor.numeric', + settings: { + foreground: '#098658', + }, + }, + { + scope: 'meta.structure.dictionary.key.python', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: ['storage.modifier', 'keyword.operator.noexcept'], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: ['string', 'meta.embedded.assembly'], + settings: { + foreground: '#A31515', + }, + }, + { + scope: [ + 'string.comment.buffered.block.pug', + 'string.quoted.pug', + 'string.interpolated.pug', + 'string.unquoted.plain.in.yaml', + 'string.unquoted.plain.out.yaml', + 'string.unquoted.block.yaml', + 'string.quoted.single.yaml', + 'string.quoted.double.xml', + 'string.quoted.single.xml', + 'string.unquoted.cdata.xml', + 'string.quoted.double.html', + 'string.quoted.single.html', + 'string.unquoted.html', + 'string.quoted.single.handlebars', + 'string.quoted.double.handlebars', + ], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#811F3F', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + ], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#000000', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: [ + 'support.type.vendored.property-name', + 'support.type.property-name', + 'variable.css', + 'variable.scss', + 'variable.other.less', + 'source.coffee.embedded', + ], + settings: { + foreground: '#E50000', + }, + }, + { + scope: ['support.type.property-name.json'], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'keyword.control', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#000000', + }, + }, + { + scope: [ + 'keyword.operator.new', + 'keyword.operator.expression', + 'keyword.operator.cast', + 'keyword.operator.sizeof', + 'keyword.operator.alignof', + 'keyword.operator.typeid', + 'keyword.operator.alignas', + 'keyword.operator.instanceof', + 'keyword.operator.logical.python', + 'keyword.operator.wordlike', + ], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#098658', + }, + }, + { + scope: ['punctuation.section.embedded.begin.php', 'punctuation.section.embedded.end.php'], + settings: { + foreground: '#800000', + }, + }, + { + scope: 'support.function.git-rebase', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'constant.sha.git-rebase', + settings: { + foreground: '#098658', + }, + }, + { + scope: ['storage.modifier.import.java', 'variable.language.wildcard.java', 'storage.modifier.package.java'], + settings: { + foreground: '#000000', + }, + }, + { + scope: 'variable.language', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: [ + 'entity.name.function', + 'support.function', + 'support.constant.handlebars', + 'source.powershell variable.other.member', + 'entity.name.operator.custom-literal', + ], + settings: { + foreground: '#795E26', + }, + }, + { + scope: [ + 'support.class', + 'support.type', + 'entity.name.type', + 'entity.name.namespace', + 'entity.other.attribute', + 'entity.name.scope-resolution', + 'entity.name.class', + 'storage.type.numeric.go', + 'storage.type.byte.go', + 'storage.type.boolean.go', + 'storage.type.string.go', + 'storage.type.uintptr.go', + 'storage.type.error.go', + 'storage.type.rune.go', + 'storage.type.cs', + 'storage.type.generic.cs', + 'storage.type.modifier.cs', + 'storage.type.variable.cs', + 'storage.type.annotation.java', + 'storage.type.generic.java', + 'storage.type.java', + 'storage.type.object.array.java', + 'storage.type.primitive.array.java', + 'storage.type.primitive.java', + 'storage.type.token.java', + 'storage.type.groovy', + 'storage.type.annotation.groovy', + 'storage.type.parameters.groovy', + 'storage.type.generic.groovy', + 'storage.type.object.array.groovy', + 'storage.type.primitive.array.groovy', + 'storage.type.primitive.groovy', + ], + settings: { + foreground: '#267F99', + }, + }, + { + scope: [ + 'meta.type.cast.expr', + 'meta.type.new.expr', + 'support.constant.math', + 'support.constant.dom', + 'support.constant.json', + 'entity.other.inherited-class', + ], + settings: { + foreground: '#267F99', + }, + }, + { + scope: [ + 'keyword.control', + 'source.cpp keyword.operator.new', + 'source.cpp keyword.operator.delete', + 'keyword.other.using', + 'keyword.other.directive.using', + 'keyword.other.operator', + 'entity.name.operator', + ], + settings: { + foreground: '#AF00DB', + }, + }, + { + scope: [ + 'variable', + 'meta.definition.variable.name', + 'support.variable', + 'entity.name.variable', + 'constant.other.placeholder', + ], + settings: { + foreground: '#001080', + }, + }, + { + scope: ['variable.other.constant', 'variable.other.enummember'], + settings: { + foreground: '#0070C1', + }, + }, + { + scope: ['meta.object-literal.key'], + settings: { + foreground: '#001080', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: [ + 'punctuation.definition.group.regexp', + 'punctuation.definition.group.assertion.regexp', + 'punctuation.definition.character-class.regexp', + 'punctuation.character.set.begin.regexp', + 'punctuation.character.set.end.regexp', + 'keyword.operator.negation.regexp', + 'support.other.parenthesis.regexp', + ], + settings: { + foreground: '#D16969', + }, + }, + { + scope: [ + 'constant.character.character-class.regexp', + 'constant.other.character-class.set.regexp', + 'constant.other.character-class.regexp', + 'constant.character.set.regexp', + ], + settings: { + foreground: '#811F3F', + }, + }, + { + scope: 'keyword.operator.quantifier.regexp', + settings: { + foreground: '#000000', + }, + }, + { + scope: ['keyword.operator.or.regexp', 'keyword.control.anchor.regexp'], + settings: { + foreground: '#EE0000', + }, + }, + { + scope: ['constant.character', 'constant.other.option'], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'constant.character.escape', + settings: { + foreground: '#EE0000', + }, + }, + { + scope: 'entity.name.label', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#316BCD', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#800080', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/monokai-dim.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/monokai-dim.ts new file mode 100644 index 0000000..384163a --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/monokai-dim.ts @@ -0,0 +1,573 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const monokaiDim: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'activityBar.background': '#353535', + 'activityBar.foreground': '#ffffff', + 'activityBarBadge.background': '#3655b5', + 'button.background': '#565656', + 'dropdown.background': '#525252', + 'editor.background': '#1e1e1e', + 'editor.foreground': '#c5c8c6', + 'editor.lineHighlightBackground': '#303030', + 'editor.selectionBackground': '#676b7180', + 'editor.selectionHighlightBackground': '#575b6180', + 'editor.wordHighlightBackground': '#4747a180', + 'editor.wordHighlightStrongBackground': '#6767ce80', + 'editorCursor.foreground': '#c07020', + 'editorGroupHeader.tabsBackground': '#282828', + 'editorIndentGuide.activeBackground': '#707057', + 'editorIndentGuide.background': '#505037', + 'editorLineNumber.activeForeground': '#949494', + 'editorWhitespace.foreground': '#505037', + focusBorder: '#3655b5', + 'inputOption.activeBorder': '#3655b5', + 'list.activeSelectionBackground': '#707070', + 'list.highlightForeground': '#e58520', + 'list.hoverBackground': '#444444', + 'list.inactiveSelectionBackground': '#4e4e4e', + 'menu.background': '#272727', + 'menu.foreground': '#cccccc', + 'minimap.selectionHighlight': '#676b7180', + 'panelTitle.activeForeground': '#ffffff', + 'peekView.border': '#3655b5', + 'pickerGroup.foreground': '#b0b0b0', + 'ports.iconRunningProcessForeground': '#cccccc', + 'quickInputList.focusBackground': '#707070', + 'sideBar.background': '#272727', + 'sideBarSectionHeader.background': '#505050', + 'statusBar.background': '#505050', + 'statusBar.debuggingBackground': '#505050', + 'statusBar.noFolderBackground': '#505050', + 'statusBarItem.remoteBackground': '#3655b5', + 'tab.border': '#303030', + 'tab.inactiveBackground': '#404040', + 'tab.inactiveForeground': '#d8d8d8', + 'tab.lastPinnedBorder': '#505050', + 'terminal.ansiBlack': '#1e1e1e', + 'terminal.ansiBlue': '#6a7ec8', + 'terminal.ansiBrightBlack': '#666666', + 'terminal.ansiBrightBlue': '#819aff', + 'terminal.ansiBrightCyan': '#66d9ef', + 'terminal.ansiBrightGreen': '#a6e22e', + 'terminal.ansiBrightMagenta': '#ae81ff', + 'terminal.ansiBrightRed': '#f92672', + 'terminal.ansiBrightWhite': '#f8f8f2', + 'terminal.ansiBrightYellow': '#e2e22e', + 'terminal.ansiCyan': '#56adbc', + 'terminal.ansiGreen': '#86b42b', + 'terminal.ansiMagenta': '#8c6bc8', + 'terminal.ansiRed': '#c4265e', + 'terminal.ansiWhite': '#e3e3dd', + 'terminal.ansiYellow': '#b3b42b', + 'terminal.inactiveSelectionBackground': '#676b7140', + 'titleBar.activeBackground': '#505050', + }, + tokenColors: [ + { + scope: ['meta.embedded', 'source.groovy.embedded', 'variable.legacy.builtin.python'], + settings: { + foreground: '#C5C8C6', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#9A9B99', + fontStyle: '', + }, + }, + { + scope: 'string', + settings: { + foreground: '#9AA83A', + fontStyle: '', + }, + }, + { + scope: 'string source', + settings: { + foreground: '#D08442', + fontStyle: '', + }, + }, + { + scope: 'constant.numeric', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#408080', + fontStyle: '', + }, + }, + { + scope: 'constant.character, constant.other', + settings: { + foreground: '#8080FF', + fontStyle: '', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'support', + settings: { + foreground: '#C7444A', + fontStyle: '', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'entity.name.class, entity.name.type, entity.name.namespace, entity.name.scope-resolution', + settings: { + foreground: '#9B0000', + fontStyle: '', + }, + }, + { + scope: 'entity.other.inherited-class', + settings: { + foreground: '#C7444A', + fontStyle: '', + }, + }, + { + scope: 'entity.name.function', + settings: { + foreground: '#CE6700', + fontStyle: '', + }, + }, + { + scope: 'variable.parameter', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'support.function', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#676867', + fontStyle: '', + }, + }, + { + scope: 'variable.other, variable.js, punctuation.separator.variable', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'punctuation.section.embedded -(source string source punctuation.section.embedded), meta.brace.erb.html', + settings: { + foreground: '#008200', + fontStyle: '', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#FF0B00', + fontStyle: '', + }, + }, + { + scope: 'variable.other.php, variable.other.normal', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'meta.function-call.object', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'variable.other.property', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: [ + 'keyword.control', + 'keyword.operator.new.cpp', + 'keyword.operator.delete.cpp', + 'keyword.other.using', + 'keyword.other.directive.using', + 'keyword.other.operator', + ], + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'meta.tag', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'meta.doctype, meta.tag.sgml-declaration.doctype, meta.tag.sgml.doctype', + settings: { + foreground: '#9AA83A', + fontStyle: '', + }, + }, + { + scope: 'meta.tag.inline source, text.html.php.source', + settings: { + foreground: '#9AA83A', + fontStyle: '', + }, + }, + { + scope: 'meta.tag.other, entity.name.tag.style, entity.name.tag.script, meta.tag.block.script, source.js.embedded punctuation.definition.tag.html, source.css.embedded punctuation.definition.tag.html', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'entity.other.attribute-name, meta.tag punctuation.definition.string', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'meta.tag string -source -punctuation, text source text meta.tag string -punctuation', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'punctuation.section.embedded -(source string source punctuation.section.embedded), meta.brace.erb.html', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'meta.toc-list.id', + settings: { + foreground: '#9AA83A', + }, + }, + { + scope: 'string.quoted.double.html, punctuation.definition.string.begin.html, punctuation.definition.string.end.html, punctuation.definition.string.end.html source, string.quoted.double.html source', + settings: { + foreground: '#9AA83A', + fontStyle: '', + }, + }, + { + scope: 'punctuation.definition.tag.html, punctuation.definition.tag.begin, punctuation.definition.tag.end', + settings: { + foreground: '#6089B4', + fontStyle: '', + }, + }, + { + scope: 'meta.selector.css entity.other.attribute-name.id', + settings: { + foreground: '#9872A2', + fontStyle: '', + }, + }, + { + scope: 'support.type.property-name.css', + settings: { + foreground: '#676867', + fontStyle: '', + }, + }, + { + scope: 'meta.property-group support.constant.property-value.css, meta.property-value support.constant.property-value.css', + settings: { + foreground: '#C7444A', + fontStyle: '', + }, + }, + { + scope: 'variable.language.js', + settings: { + foreground: '#CC555A', + }, + }, + { + scope: ['punctuation.definition.template-expression', 'punctuation.section.embedded.coffee'], + settings: { + foreground: '#D08442', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#C5C8C6', + }, + }, + { + scope: 'meta.function-call.object.php', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'punctuation.definition.string.end.php, punctuation.definition.string.begin.php', + settings: { + foreground: '#9AA83A', + }, + }, + { + scope: 'source.php.embedded.line.html', + settings: { + foreground: '#676867', + }, + }, + { + scope: 'punctuation.section.embedded.begin.php, punctuation.section.embedded.end.php', + settings: { + foreground: '#D08442', + fontStyle: '', + }, + }, + { + scope: 'constant.other.symbol.ruby', + settings: { + foreground: '#9AA83A', + fontStyle: '', + }, + }, + { + scope: 'variable.language.ruby', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'keyword.other.special-method.ruby', + settings: { + foreground: '#D9B700', + fontStyle: '', + }, + }, + { + scope: ['punctuation.section.embedded.begin.ruby', 'punctuation.section.embedded.end.ruby'], + settings: { + foreground: '#D08442', + }, + }, + { + scope: 'keyword.other.DML.sql', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'meta.diff, meta.diff.header', + settings: { + foreground: '#E0EDDD', + fontStyle: 'italic', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#DC322F', + fontStyle: '', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#CB4B16', + fontStyle: '', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#219186', + }, + }, + { + scope: 'markup.quote', + settings: { + foreground: '#9872A2', + }, + }, + { + scope: 'markup.list', + settings: { + foreground: '#9AA83A', + }, + }, + { + scope: 'markup.bold, markup.italic', + settings: { + foreground: '#6089B4', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#FF0080', + fontStyle: '', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#D0B344', + }, + }, + { + scope: 'markup.heading.setext', + settings: { + foreground: '#D0B344', + fontStyle: '', + }, + }, + { + scope: 'markup.heading.markdown', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.quote.markdown', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.bold.markdown', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'string.other.link.title.markdown,string.other.link.description.markdown', + settings: { + foreground: '#AE81FF', + }, + }, + { + scope: 'markup.underline.link.markdown,markup.underline.link.image.markdown', + settings: {}, + }, + { + scope: 'markup.italic.markdown', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.list.unnumbered.markdown, markup.list.numbered.markdown', + settings: {}, + }, + { + scope: ['punctuation.definition.list.begin.markdown'], + settings: {}, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + { + scope: 'variable.language', + settings: { + foreground: '#C7444A', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/quiet-light.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/quiet-light.ts new file mode 100644 index 0000000..61a94e0 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/quiet-light.ts @@ -0,0 +1,465 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const quietLight: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'light', + colors: { + 'activityBar.background': '#ededf5', + 'activityBar.foreground': '#705697', + 'activityBarBadge.background': '#705697', + 'badge.background': '#705697aa', + 'button.background': '#705697', + 'dropdown.background': '#f5f5f5', + 'editor.background': '#f5f5f5', + 'editor.findMatchBackground': '#bf9cac', + 'editor.findMatchHighlightBackground': '#edc9d899', + 'editor.lineHighlightBackground': '#e4f6d4', + 'editor.selectionBackground': '#c9d0d9', + 'editorCursor.foreground': '#54494b', + 'editorGroup.dropBackground': '#c9d0d988', + 'editorIndentGuide.activeBackground': '#777777b0', + 'editorIndentGuide.background': '#aaaaaa60', + 'editorLineNumber.activeForeground': '#9769dc', + 'editorLineNumber.foreground': '#6d705b', + 'editorWhitespace.foreground': '#aaaaaa', + errorForeground: '#f1897f', + focusBorder: '#9769dc', + 'inputOption.activeBorder': '#adafb7', + 'inputValidation.errorBackground': '#ffeaea', + 'inputValidation.errorBorder': '#f1897f', + 'inputValidation.infoBackground': '#f2fcff', + 'inputValidation.infoBorder': '#4ec1e5', + 'inputValidation.warningBackground': '#fffee2', + 'inputValidation.warningBorder': '#ffe055', + 'list.activeSelectionBackground': '#c4d9b1', + 'list.activeSelectionForeground': '#6c6c6c', + 'list.highlightForeground': '#9769dc', + 'list.hoverBackground': '#e0e0e0', + 'list.inactiveSelectionBackground': '#d3dbcd', + 'minimap.selectionHighlight': '#c9d0d9', + 'panel.background': '#f5f5f5', + 'peekView.border': '#705697', + 'peekViewEditor.background': '#f2f8fc', + 'peekViewEditor.matchHighlightBackground': '#c2dfe3', + 'peekViewResult.background': '#f2f8fc', + 'peekViewResult.matchHighlightBackground': '#93c6d6', + 'peekViewTitle.background': '#f2f8fc', + 'pickerGroup.border': '#749351', + 'pickerGroup.foreground': '#a6b39b', + 'ports.iconRunningProcessForeground': '#749351', + 'progressBar.background': '#705697', + 'quickInputList.focusBackground': '#cadeb9', + 'selection.background': '#c9d0d9', + 'sideBar.background': '#f2f2f2', + 'sideBarSectionHeader.background': '#ede8ef', + 'statusBar.background': '#705697', + 'statusBar.debuggingBackground': '#705697', + 'statusBar.noFolderBackground': '#705697', + 'statusBarItem.remoteBackground': '#4e3c69', + 'tab.lastPinnedBorder': '#c9d0d9', + 'titleBar.activeBackground': '#c4b7d7', + 'walkThrough.embeddedEditorBackground': '#00000014', + 'welcomePage.tileBackground': '#f0f0f7', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#333333', + }, + }, + { + scope: ['comment', 'punctuation.definition.comment'], + settings: { + foreground: '#AAAAAA', + fontStyle: 'italic', + }, + }, + { + scope: 'comment.block.preprocessor', + settings: { + foreground: '#AAAAAA', + fontStyle: '', + }, + }, + { + scope: [ + 'comment.documentation', + 'comment.block.documentation', + 'comment.block.documentation punctuation.definition.comment ', + ], + settings: { + foreground: '#448C27', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'invalid.illegal', + settings: { + foreground: '#660000', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#777777', + }, + }, + { + scope: ['keyword', 'storage'], + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: ['storage.type', 'support.type'], + settings: { + foreground: '#7A3E9D', + }, + }, + { + scope: ['constant.language', 'support.constant', 'variable.language'], + settings: { + foreground: '#9C5D27', + }, + }, + { + scope: ['variable', 'support.variable'], + settings: { + foreground: '#7A3E9D', + }, + }, + { + scope: ['entity.name.function', 'support.function'], + settings: { + foreground: '#AA3731', + fontStyle: 'bold', + }, + }, + { + scope: [ + 'entity.name.type', + 'entity.name.namespace', + 'entity.name.scope-resolution', + 'entity.other.inherited-class', + 'support.class', + ], + settings: { + foreground: '#7A3E9D', + fontStyle: 'bold', + }, + }, + { + scope: 'entity.name.exception', + settings: { + foreground: '#660000', + }, + }, + { + scope: 'entity.name.section', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: ['constant.numeric', 'constant.character', 'constant'], + settings: { + foreground: '#9C5D27', + }, + }, + { + scope: 'string', + settings: { + foreground: '#448C27', + }, + }, + { + scope: 'constant.character.escape', + settings: { + foreground: '#777777', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: 'constant.other.symbol', + settings: { + foreground: '#9C5D27', + }, + }, + { + scope: 'punctuation', + settings: { + foreground: '#777777', + }, + }, + { + scope: [ + 'meta.tag.sgml.doctype', + 'meta.tag.sgml.doctype string', + 'meta.tag.sgml.doctype entity.name.tag', + 'meta.tag.sgml punctuation.definition.tag.html', + ], + settings: { + foreground: '#AAAAAA', + }, + }, + { + scope: [ + 'meta.tag', + 'punctuation.definition.tag.html', + 'punctuation.definition.tag.begin.html', + 'punctuation.definition.tag.end.html', + ], + settings: { + foreground: '#91B3E0', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: ['meta.tag entity.other.attribute-name', 'entity.other.attribute-name.html'], + settings: { + foreground: '#8190A0', + fontStyle: 'italic', + }, + }, + { + scope: ['constant.character.entity', 'punctuation.definition.entity'], + settings: { + foreground: '#9C5D27', + }, + }, + { + scope: ['meta.selector', 'meta.selector entity', 'meta.selector entity punctuation', 'entity.name.tag.css'], + settings: { + foreground: '#7A3E9D', + }, + }, + { + scope: ['meta.property-name', 'support.type.property-name'], + settings: { + foreground: '#9C5D27', + }, + }, + { + scope: ['meta.property-value', 'meta.property-value constant.other', 'support.constant.property-value'], + settings: { + foreground: '#448C27', + }, + }, + { + scope: 'keyword.other.important', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.error', + settings: { + foreground: '#660000', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'meta.link', + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: ['markup.output', 'markup.raw'], + settings: { + foreground: '#777777', + }, + }, + { + scope: 'markup.prompt', + settings: { + foreground: '#777777', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#AA3731', + }, + }, + { + scope: 'markup.bold', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.traceback', + settings: { + foreground: '#660000', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.quote', + settings: { + foreground: '#7A3E9D', + }, + }, + { + scope: 'markup.list', + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: ['markup.bold', 'markup.italic'], + settings: { + foreground: '#448C27', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#9C5D27', + fontStyle: '', + }, + }, + { + scope: ['meta.diff.range', 'meta.diff.index', 'meta.separator'], + settings: { + foreground: '#434343', + }, + }, + { + scope: ['meta.diff.header.from-file', 'punctuation.definition.from-file.diff'], + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: ['meta.diff.header.to-file', 'punctuation.definition.to-file.diff'], + settings: { + foreground: '#4B69C6', + }, + }, + { + scope: 'markup.deleted.diff', + settings: { + foreground: '#C73D20', + }, + }, + { + scope: 'markup.changed.diff', + settings: { + foreground: '#9C5D27', + }, + }, + { + scope: 'markup.inserted.diff', + settings: { + foreground: '#448C27', + }, + }, + { + scope: [ + 'punctuation.definition.tag.js', + 'punctuation.definition.tag.begin.js', + 'punctuation.definition.tag.end.js', + ], + settings: { + foreground: '#91B3E0', + }, + }, + { + scope: 'meta.jsx.children.js', + settings: { + foreground: '#333333', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#316BCD', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#800080', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/red.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/red.ts new file mode 100644 index 0000000..0b50970 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/red.ts @@ -0,0 +1,376 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const red: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'activityBar.background': '#580000', + 'badge.background': '#cc3333', + 'button.background': '#883333', + 'debugToolBar.background': '#660000', + 'dropdown.background': '#580000', + 'editor.background': '#390000', + 'editor.foreground': '#f8f8f8', + 'editor.hoverHighlightBackground': '#ff000044', + 'editor.lineHighlightBackground': '#ff000033', + 'editor.selectionBackground': '#750000', + 'editor.selectionHighlightBackground': '#f5500039', + 'editorCursor.foreground': '#970000', + 'editorGroup.border': '#ff666633', + 'editorGroupHeader.tabsBackground': '#330000', + 'editorHoverWidget.background': '#300000', + 'editorLineNumber.activeForeground': '#ffbbbb88', + 'editorLineNumber.foreground': '#ff777788', + 'editorLink.activeForeground': '#ffd0aa', + 'editorSuggestWidget.background': '#300000', + 'editorSuggestWidget.border': '#220000', + 'editorWhitespace.foreground': '#c10000', + 'editorWidget.background': '#300000', + errorForeground: '#ffeaea', + 'extensionButton.prominentBackground': '#cc3333', + 'extensionButton.prominentHoverBackground': '#cc333388', + focusBorder: '#ff6666aa', + 'input.background': '#580000', + 'inputOption.activeBorder': '#cc0000', + 'inputValidation.infoBackground': '#550000', + 'inputValidation.infoBorder': '#db7e58', + 'list.activeSelectionBackground': '#880000', + 'list.dropBackground': '#662222', + 'list.highlightForeground': '#ff4444', + 'list.hoverBackground': '#800000', + 'list.inactiveSelectionBackground': '#770000', + 'minimap.selectionHighlight': '#750000', + 'peekView.border': '#ff000044', + 'peekViewEditor.background': '#300000', + 'peekViewResult.background': '#400000', + 'peekViewTitle.background': '#550000', + 'pickerGroup.border': '#ff000033', + 'pickerGroup.foreground': '#cc9999', + 'ports.iconRunningProcessForeground': '#db7e58', + 'progressBar.background': '#cc3333', + 'quickInputList.focusBackground': '#660000', + 'selection.background': '#ff777788', + 'sideBar.background': '#330000', + 'statusBar.background': '#700000', + 'statusBar.noFolderBackground': '#700000', + 'statusBarItem.remoteBackground': '#cc3333', + 'tab.activeBackground': '#490000', + 'tab.inactiveBackground': '#300a0a', + 'tab.lastPinnedBorder': '#ff000044', + 'titleBar.activeBackground': '#770000', + 'titleBar.inactiveBackground': '#772222', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#F8F8F8', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#E7C0C0', + fontStyle: 'italic', + }, + }, + { + scope: 'constant', + settings: { + foreground: '#994646', + fontStyle: '', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#F12727', + fontStyle: '', + }, + }, + { + scope: 'entity', + settings: { + foreground: '#FEC758', + fontStyle: '', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#FF6262', + fontStyle: 'bold', + }, + }, + { + scope: 'string', + settings: { + foreground: '#CD8D8D', + fontStyle: '', + }, + }, + { + scope: 'support', + settings: { + foreground: '#9DF39F', + fontStyle: '', + }, + }, + { + scope: 'variable', + settings: { + foreground: '#FB9A4B', + fontStyle: 'italic', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'entity.other.inherited-class', + settings: { + foreground: '#AA5507', + fontStyle: 'underline', + }, + }, + { + scope: 'constant.character', + settings: { + foreground: '#EC0D1E', + }, + }, + { + scope: ['string constant', 'constant.character.escape'], + settings: { + foreground: '#FFE862', + fontStyle: '', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#FFB454', + }, + }, + { + scope: 'string variable', + settings: { + foreground: '#EDEF7D', + }, + }, + { + scope: 'support.function', + settings: { + foreground: '#FFB454', + fontStyle: '', + }, + }, + { + scope: ['support.constant', 'support.variable'], + settings: { + foreground: '#EB939A', + fontStyle: '', + }, + }, + { + scope: [ + 'declaration.sgml.html declaration.doctype', + 'declaration.sgml.html declaration.doctype entity', + 'declaration.sgml.html declaration.doctype string', + 'declaration.xml-processing', + 'declaration.xml-processing entity', + 'declaration.xml-processing string', + ], + settings: { + foreground: '#73817D', + fontStyle: '', + }, + }, + { + scope: ['declaration.tag', 'declaration.tag entity', 'meta.tag', 'meta.tag entity'], + settings: { + foreground: '#EC0D1E', + fontStyle: '', + }, + }, + { + scope: 'meta.selector.css entity.name.tag', + settings: { + foreground: '#AA5507', + fontStyle: '', + }, + }, + { + scope: 'meta.selector.css entity.other.attribute-name.id', + settings: { + foreground: '#FEC758', + }, + }, + { + scope: 'meta.selector.css entity.other.attribute-name.class', + settings: { + foreground: '#41A83E', + fontStyle: '', + }, + }, + { + scope: 'support.type.property-name.css', + settings: { + foreground: '#96DD3B', + fontStyle: '', + }, + }, + { + scope: [ + 'meta.property-group support.constant.property-value.css', + 'meta.property-value support.constant.property-value.css', + ], + settings: { + foreground: '#FFE862', + fontStyle: 'italic', + }, + }, + { + scope: ['meta.property-value support.constant.named-color.css', 'meta.property-value constant'], + settings: { + foreground: '#FFE862', + fontStyle: '', + }, + }, + { + scope: 'meta.preprocessor.at-rule keyword.control.at-rule', + settings: { + foreground: '#FD6209', + }, + }, + { + scope: 'meta.constructor.argument.css', + settings: { + foreground: '#EC9799', + fontStyle: '', + }, + }, + { + scope: ['meta.diff', 'meta.diff.header'], + settings: { + foreground: '#F8F8F8', + fontStyle: 'italic', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#EC9799', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#F8F8F8', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#41A83E', + }, + }, + { + scope: 'markup.quote', + settings: { + foreground: '#F12727', + }, + }, + { + scope: 'markup.list', + settings: { + foreground: '#FF6262', + }, + }, + { + scope: ['markup.bold', 'markup.italic'], + settings: { + foreground: '#FB9A4B', + }, + }, + { + scope: 'markup.bold', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#CD8D8D', + fontStyle: '', + }, + }, + { + scope: ['markup.heading', 'markup.heading.setext', 'punctuation.definition.heading', 'entity.name.section'], + settings: { + foreground: '#FEC758', + fontStyle: 'bold', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + '.format.placeholder', + ], + settings: { + foreground: '#EC0D1E', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/tomorrow-night-blue.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/tomorrow-night-blue.ts new file mode 100644 index 0000000..56e7444 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/tomorrow-night-blue.ts @@ -0,0 +1,265 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const tomorrowNightBlue: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'activityBar.background': '#001733', + 'badge.background': '#bbdaffcc', + 'badge.foreground': '#001733', + 'debugToolBar.background': '#001c40', + 'dropdown.background': '#001733', + 'editor.background': '#002451', + 'editor.foreground': '#ffffff', + 'editor.lineHighlightBackground': '#00346e', + 'editor.selectionBackground': '#003f8e', + 'editorCursor.foreground': '#ffffff', + 'editorGroup.border': '#404f7d', + 'editorGroup.dropBackground': '#25375daa', + 'editorGroupHeader.tabsBackground': '#001733', + 'editorHoverWidget.background': '#001c40', + 'editorHoverWidget.border': '#ffffff44', + 'editorLineNumber.activeForeground': '#949494', + 'editorWhitespace.foreground': '#404f7d', + 'editorWidget.background': '#001c40', + errorForeground: '#a92049', + focusBorder: '#bbdaff', + 'input.background': '#001733', + 'list.activeSelectionBackground': '#ffffff60', + 'list.highlightForeground': '#bbdaff', + 'list.hoverBackground': '#ffffff30', + 'list.inactiveSelectionBackground': '#ffffff40', + 'minimap.selectionHighlight': '#003f8e', + 'peekViewResult.background': '#001c40', + 'pickerGroup.foreground': '#bbdaff', + 'ports.iconRunningProcessForeground': '#bbdaff', + 'progressBar.background': '#bbdaffcc', + 'quickInputList.focusBackground': '#ffffff60', + 'sideBar.background': '#001c40', + 'statusBar.background': '#001126', + 'statusBar.debuggingBackground': '#001126', + 'statusBar.noFolderBackground': '#001126', + 'statusBarItem.remoteBackground': '#0e639c', + 'tab.inactiveBackground': '#001c40', + 'tab.lastPinnedBorder': '#007acc80', + 'terminal.ansiBlack': '#111111', + 'terminal.ansiBlue': '#bbdaff', + 'terminal.ansiBrightBlack': '#333333', + 'terminal.ansiBrightBlue': '#80baff', + 'terminal.ansiBrightCyan': '#78ffff', + 'terminal.ansiBrightGreen': '#b8f171', + 'terminal.ansiBrightMagenta': '#d778ff', + 'terminal.ansiBrightRed': '#ff7882', + 'terminal.ansiBrightWhite': '#ffffff', + 'terminal.ansiBrightYellow': '#ffe580', + 'terminal.ansiCyan': '#99ffff', + 'terminal.ansiGreen': '#d1f1a9', + 'terminal.ansiMagenta': '#ebbbff', + 'terminal.ansiRed': '#ff9da4', + 'terminal.ansiWhite': '#cccccc', + 'terminal.ansiYellow': '#ffeead', + 'titleBar.activeBackground': '#001126', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'meta.jsx.children', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#7285B7', + }, + }, + { + scope: 'keyword.operator.class, keyword.operator, constant.other, source.php.embedded.line', + settings: { + foreground: '#FFFFFF', + fontStyle: '', + }, + }, + { + scope: 'variable, support.other.variable, string.other.link, string.regexp, entity.name.tag, entity.other.attribute-name, meta.tag, declaration.tag, markup.deleted.git_gutter', + settings: { + foreground: '#FF9DA4', + }, + }, + { + scope: 'constant.numeric, constant.language, support.constant, constant.character, variable.parameter, punctuation.section.embedded, keyword.other.unit', + settings: { + foreground: '#FFC58F', + fontStyle: '', + }, + }, + { + scope: 'entity.name.class, entity.name.type, entity.name.namespace, entity.name.scope-resolution, support.type, support.class', + settings: { + foreground: '#FFEEAD', + fontStyle: '', + }, + }, + { + scope: 'string, constant.other.symbol, entity.other.inherited-class, markup.heading, markup.inserted.git_gutter', + settings: { + foreground: '#D1F1A9', + fontStyle: '', + }, + }, + { + scope: 'keyword.operator, constant.other.color', + settings: { + foreground: '#99FFFF', + }, + }, + { + scope: 'entity.name.function, meta.function-call, support.function, keyword.other.special-method, meta.block-level, markup.changed.git_gutter', + settings: { + foreground: '#BBDAFF', + fontStyle: '', + }, + }, + { + scope: 'keyword, storage, storage.type, entity.name.tag.css', + settings: { + foreground: '#EBBBFF', + fontStyle: '', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#A92049', + fontStyle: '', + }, + }, + { + scope: 'meta.separator', + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'invalid.deprecated', + settings: { + foreground: '#CD9731', + fontStyle: '', + }, + }, + { + scope: 'markup.inserted.diff, markup.deleted.diff, meta.diff.header.to-file, meta.diff.header.from-file', + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'markup.inserted.diff, meta.diff.header.to-file', + settings: { + foreground: '#718C00', + }, + }, + { + scope: 'markup.deleted.diff, meta.diff.header.from-file', + settings: { + foreground: '#C82829', + }, + }, + { + scope: 'meta.diff.header.from-file, meta.diff.header.to-file', + settings: { + foreground: '#4271AE', + }, + }, + { + scope: 'meta.diff.range', + settings: { + foreground: '#3E999F', + fontStyle: 'italic', + }, + }, + { + scope: 'markup.quote', + settings: { + foreground: '#FFC58F', + }, + }, + { + scope: 'markup.list', + settings: { + foreground: '#BBDAFF', + }, + }, + { + scope: 'markup.bold, markup.italic', + settings: { + foreground: '#FFC58F', + }, + }, + { + scope: 'markup.bold', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#FF9DA4', + fontStyle: '', + }, + }, + { + scope: 'markup.heading', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/vs-dark.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/vs-dark.ts new file mode 100644 index 0000000..8bbf36b --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/vs-dark.ts @@ -0,0 +1,412 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const vsDark: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'dark', + colors: { + 'actionBar.toggledBackground': '#383a49', + 'activityBarBadge.background': '#007acc', + 'checkbox.border': '#6b6b6b', + 'editor.background': '#1e1e1e', + 'editor.foreground': '#d4d4d4', + 'editor.inactiveSelectionBackground': '#3a3d41', + 'editor.selectionHighlightBackground': '#add6ff26', + 'editorIndentGuide.activeBackground1': '#707070', + 'editorIndentGuide.background1': '#404040', + 'input.placeholderForeground': '#a6a6a6', + 'list.activeSelectionIconForeground': '#ffffff', + 'list.dropBackground': '#383b3d', + 'menu.background': '#252526', + 'menu.border': '#454545', + 'menu.foreground': '#cccccc', + 'menu.separatorBackground': '#454545', + 'ports.iconRunningProcessForeground': '#369432', + 'sideBarSectionHeader.background': '#00000000', + 'sideBarSectionHeader.border': '#cccccc33', + 'sideBarTitle.foreground': '#bbbbbb', + 'statusBarItem.remoteBackground': '#16825d', + 'statusBarItem.remoteForeground': '#ffffff', + 'tab.lastPinnedBorder': '#cccccc33', + 'terminal.inactiveSelectionBackground': '#3a3d41', + 'widget.border': '#303031', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'emphasis', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'strong', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'header', + settings: { + foreground: '#000080', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#6A9955', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: [ + 'constant.numeric', + 'variable.other.enummember', + 'keyword.operator.plus.exponent', + 'keyword.operator.minus.exponent', + ], + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'constant.regexp', + settings: { + foreground: '#646695', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'entity.name.tag.css', + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: [ + 'entity.other.attribute-name.class.css', + 'entity.other.attribute-name.class.mixin.css', + 'entity.other.attribute-name.id.css', + 'entity.other.attribute-name.parent-selector.css', + 'entity.other.attribute-name.pseudo-class.css', + 'entity.other.attribute-name.pseudo-element.css', + 'source.css.less entity.other.attribute-name.id', + 'entity.other.attribute-name.scss', + ], + settings: { + foreground: '#D7BA7D', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.bold', + settings: { + foreground: '#569CD6', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#569CD6', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'punctuation.definition.quote.begin.markdown', + settings: { + foreground: '#6A9955', + }, + }, + { + scope: 'punctuation.definition.list.begin.markdown', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'punctuation.definition.tag', + settings: { + foreground: '#808080', + }, + }, + { + scope: ['meta.preprocessor', 'entity.name.function.preprocessor'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'meta.preprocessor.string', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'meta.preprocessor.numeric', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: 'meta.structure.dictionary.key.python', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'meta.diff.header', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['storage.modifier', 'keyword.operator.noexcept'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['string', 'meta.embedded.assembly'], + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.tag', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.value', + settings: { + foreground: '#CE9178', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#D16969', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + ], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: [ + 'support.type.vendored.property-name', + 'support.type.property-name', + 'variable.css', + 'variable.scss', + 'variable.other.less', + 'source.coffee.embedded', + ], + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.control', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: [ + 'keyword.operator.new', + 'keyword.operator.expression', + 'keyword.operator.cast', + 'keyword.operator.sizeof', + 'keyword.operator.alignof', + 'keyword.operator.typeid', + 'keyword.operator.alignas', + 'keyword.operator.instanceof', + 'keyword.operator.logical.python', + 'keyword.operator.wordlike', + ], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: ['punctuation.section.embedded.begin.php', 'punctuation.section.embedded.end.php'], + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'support.function.git-rebase', + settings: { + foreground: '#9CDCFE', + }, + }, + { + scope: 'constant.sha.git-rebase', + settings: { + foreground: '#B5CEA8', + }, + }, + { + scope: ['storage.modifier.import.java', 'variable.language.wildcard.java', 'storage.modifier.package.java'], + settings: { + foreground: '#D4D4D4', + }, + }, + { + scope: 'variable.language', + settings: { + foreground: '#569CD6', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#FFFFFF', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#6796E6', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#F44747', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#B267E6', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/themes/vs-light.ts b/completions-sample-code/vscode-node/extension/src/panelShared/themes/vs-light.ts new file mode 100644 index 0000000..6e7f6f7 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/themes/vs-light.ts @@ -0,0 +1,435 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ThemeRegistrationAny } from 'shiki'; + +export const vsLight: ThemeRegistrationAny = { + $schema: 'vscode://schemas/color-theme', + type: 'light', + colors: { + 'actionBar.toggledBackground': '#dddddd', + 'activityBarBadge.background': '#007acc', + 'checkbox.border': '#919191', + 'editor.background': '#ffffff', + 'editor.foreground': '#000000', + 'editor.inactiveSelectionBackground': '#e5ebf1', + 'editor.selectionHighlightBackground': '#add6ff80', + 'editorIndentGuide.activeBackground1': '#939393', + 'editorIndentGuide.background1': '#d3d3d3', + 'editorSuggestWidget.background': '#f3f3f3', + 'input.placeholderForeground': '#767676', + 'list.activeSelectionIconForeground': '#ffffff', + 'list.focusAndSelectionOutline': '#90c2f9', + 'list.hoverBackground': '#e8e8e8', + 'menu.border': '#d4d4d4', + 'notebook.cellBorderColor': '#e8e8e8', + 'notebook.selectedCellBackground': '#c8ddf150', + 'ports.iconRunningProcessForeground': '#369432', + 'searchEditor.textInputBorder': '#cecece', + 'settings.numberInputBorder': '#cecece', + 'settings.textInputBorder': '#cecece', + 'sideBarSectionHeader.background': '#00000000', + 'sideBarSectionHeader.border': '#61616130', + 'sideBarTitle.foreground': '#6f6f6f', + 'statusBarItem.errorBackground': '#c72e0f', + 'statusBarItem.remoteBackground': '#16825d', + 'statusBarItem.remoteForeground': '#ffffff', + 'tab.lastPinnedBorder': '#61616130', + 'terminal.inactiveSelectionBackground': '#e5ebf1', + 'widget.border': '#d4d4d4', + }, + tokenColors: [ + { + scope: [ + 'meta.embedded', + 'source.groovy.embedded', + 'string meta.image.inline.markdown', + 'variable.legacy.builtin.python', + ], + settings: { + foreground: '#000000', + }, + }, + { + scope: 'emphasis', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'strong', + settings: { + fontStyle: 'bold', + }, + }, + { + scope: 'meta.diff.header', + settings: { + foreground: '#000080', + }, + }, + { + scope: 'comment', + settings: { + foreground: '#008000', + }, + }, + { + scope: 'constant.language', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: [ + 'constant.numeric', + 'variable.other.enummember', + 'keyword.operator.plus.exponent', + 'keyword.operator.minus.exponent', + ], + settings: { + foreground: '#098658', + }, + }, + { + scope: 'constant.regexp', + settings: { + foreground: '#811F3F', + }, + }, + { + scope: 'entity.name.tag', + settings: { + foreground: '#800000', + }, + }, + { + scope: 'entity.name.selector', + settings: { + foreground: '#800000', + }, + }, + { + scope: 'entity.other.attribute-name', + settings: { + foreground: '#E50000', + }, + }, + { + scope: [ + 'entity.other.attribute-name.class.css', + 'entity.other.attribute-name.class.mixin.css', + 'entity.other.attribute-name.id.css', + 'entity.other.attribute-name.parent-selector.css', + 'entity.other.attribute-name.pseudo-class.css', + 'entity.other.attribute-name.pseudo-element.css', + 'source.css.less entity.other.attribute-name.id', + 'entity.other.attribute-name.scss', + ], + settings: { + foreground: '#800000', + }, + }, + { + scope: 'invalid', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'markup.underline', + settings: { + fontStyle: 'underline', + }, + }, + { + scope: 'markup.bold', + settings: { + foreground: '#000080', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.heading', + settings: { + foreground: '#800000', + fontStyle: 'bold', + }, + }, + { + scope: 'markup.italic', + settings: { + fontStyle: 'italic', + }, + }, + { + scope: 'markup.strikethrough', + settings: { + fontStyle: 'strikethrough', + }, + }, + { + scope: 'markup.inserted', + settings: { + foreground: '#098658', + }, + }, + { + scope: 'markup.deleted', + settings: { + foreground: '#A31515', + }, + }, + { + scope: 'markup.changed', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: ['punctuation.definition.quote.begin.markdown', 'punctuation.definition.list.begin.markdown'], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'markup.inline.raw', + settings: { + foreground: '#800000', + }, + }, + { + scope: 'punctuation.definition.tag', + settings: { + foreground: '#800000', + }, + }, + { + scope: ['meta.preprocessor', 'entity.name.function.preprocessor'], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'meta.preprocessor.string', + settings: { + foreground: '#A31515', + }, + }, + { + scope: 'meta.preprocessor.numeric', + settings: { + foreground: '#098658', + }, + }, + { + scope: 'meta.structure.dictionary.key.python', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'storage', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'storage.type', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: ['storage.modifier', 'keyword.operator.noexcept'], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: ['string', 'meta.embedded.assembly'], + settings: { + foreground: '#A31515', + }, + }, + { + scope: [ + 'string.comment.buffered.block.pug', + 'string.quoted.pug', + 'string.interpolated.pug', + 'string.unquoted.plain.in.yaml', + 'string.unquoted.plain.out.yaml', + 'string.unquoted.block.yaml', + 'string.quoted.single.yaml', + 'string.quoted.double.xml', + 'string.quoted.single.xml', + 'string.unquoted.cdata.xml', + 'string.quoted.double.html', + 'string.quoted.single.html', + 'string.unquoted.html', + 'string.quoted.single.handlebars', + 'string.quoted.double.handlebars', + ], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'string.regexp', + settings: { + foreground: '#811F3F', + }, + }, + { + scope: [ + 'punctuation.definition.template-expression.begin', + 'punctuation.definition.template-expression.end', + 'punctuation.section.embedded', + ], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: ['meta.template.expression'], + settings: { + foreground: '#000000', + }, + }, + { + scope: [ + 'support.constant.property-value', + 'support.constant.font-name', + 'support.constant.media-type', + 'support.constant.media', + 'constant.other.color.rgb-value', + 'constant.other.rgb-value', + 'support.constant.color', + ], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: [ + 'support.type.vendored.property-name', + 'support.type.property-name', + 'variable.css', + 'variable.scss', + 'variable.other.less', + 'source.coffee.embedded', + ], + settings: { + foreground: '#E50000', + }, + }, + { + scope: ['support.type.property-name.json'], + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'keyword', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'keyword.control', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'keyword.operator', + settings: { + foreground: '#000000', + }, + }, + { + scope: [ + 'keyword.operator.new', + 'keyword.operator.expression', + 'keyword.operator.cast', + 'keyword.operator.sizeof', + 'keyword.operator.alignof', + 'keyword.operator.typeid', + 'keyword.operator.alignas', + 'keyword.operator.instanceof', + 'keyword.operator.logical.python', + 'keyword.operator.wordlike', + ], + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'keyword.other.unit', + settings: { + foreground: '#098658', + }, + }, + { + scope: ['punctuation.section.embedded.begin.php', 'punctuation.section.embedded.end.php'], + settings: { + foreground: '#800000', + }, + }, + { + scope: 'support.function.git-rebase', + settings: { + foreground: '#0451A5', + }, + }, + { + scope: 'constant.sha.git-rebase', + settings: { + foreground: '#098658', + }, + }, + { + scope: ['storage.modifier.import.java', 'variable.language.wildcard.java', 'storage.modifier.package.java'], + settings: { + foreground: '#000000', + }, + }, + { + scope: 'variable.language', + settings: { + foreground: '#0000FF', + }, + }, + { + scope: 'ref.matchtext', + settings: { + foreground: '#000000', + }, + }, + { + scope: 'token.info-token', + settings: { + foreground: '#316BCD', + }, + }, + { + scope: 'token.warn-token', + settings: { + foreground: '#CD9731', + }, + }, + { + scope: 'token.error-token', + settings: { + foreground: '#CD3131', + }, + }, + { + scope: 'token.debug-token', + settings: { + foreground: '#800080', + }, + }, + ], +}; diff --git a/completions-sample-code/vscode-node/extension/src/panelShared/utils.ts b/completions-sample-code/vscode-node/extension/src/panelShared/utils.ts new file mode 100644 index 0000000..7117c46 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/panelShared/utils.ts @@ -0,0 +1,17 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export function getNonce() { + let text = ''; + const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + for (let i = 0; i < 32; i++) { + text += possible.charAt(Math.floor(Math.random() * possible.length)); + } + return text; +} + +export function pluralize(count: number, noun: string, suffix = 's') { + return `${count} ${noun}${count !== 1 ? suffix : ''}`; +} diff --git a/completions-sample-code/vscode-node/extension/src/statusBar.ts b/completions-sample-code/vscode-node/extension/src/statusBar.ts new file mode 100644 index 0000000..b936070 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/statusBar.ts @@ -0,0 +1,134 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { commands, Disposable, languages, LanguageStatusItem, LanguageStatusSeverity, window, workspace } from 'vscode'; +import { IDisposable } from '../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotConfigPrefix } from '../../lib/src/constants'; +import { CMDQuotaExceeded } from '../../lib/src/openai/fetch'; +import { StatusChangedEvent, StatusReporter } from '../../lib/src/progress'; +import { isCompletionEnabled, isInlineSuggestEnabled } from './config'; +import { CMDToggleStatusMenuChat } from './constants'; +import { ICompletionsExtensionStatus } from './extensionStatus'; +import { Icon } from './icon'; + +export class CopilotStatusBar extends StatusReporter implements IDisposable { + readonly item!: LanguageStatusItem; + showingMessage = false; + private disposables: Disposable[] = []; + + constructor( + id: string, + @ICompletionsExtensionStatus readonly extensionStatusService: ICompletionsExtensionStatus, + @IInstantiationService readonly instantiationService: IInstantiationService, + + ) { + super(); + + this.item = languages.createLanguageStatusItem(id, '*'); + this.disposables.push(this.item); + + this.updateStatusBarIndicator(); + + this.disposables.push( + window.onDidChangeActiveTextEditor(() => { + this.updateStatusBarIndicator(); + }) + ); + + this.disposables.push( + workspace.onDidCloseTextDocument(() => { + this.updateStatusBarIndicator(); + }) + ); + + this.disposables.push( + workspace.onDidOpenTextDocument(() => { + this.updateStatusBarIndicator(); + }) + ); + + this.disposables.push( + workspace.onDidChangeConfiguration(e => { + if (!e.affectsConfiguration(CopilotConfigPrefix)) { return; } + this.updateStatusBarIndicator(); + }) + ); + } + + override didChange(event: StatusChangedEvent): void { + this.extensionStatusService.kind = event.kind; + this.extensionStatusService.message = event.message; + this.extensionStatusService.command = event.command; + this.updateStatusBarIndicator(); + } + + private checkEnabledForLanguage(): boolean { + return this.instantiationService.invokeFunction(isCompletionEnabled) ?? true; + } + + protected updateStatusBarIndicator() { + if (this.isDisposed()) { + return; + } + void commands.executeCommand( + 'setContext', + 'github.copilot.completions.quotaExceeded', + this.extensionStatusService.command?.command === CMDQuotaExceeded + ); + const enabled = this.checkEnabledForLanguage(); + void commands.executeCommand('setContext', 'github.copilot.completions.enabled', enabled); + this.item.command = { command: CMDToggleStatusMenuChat, title: 'View Details' }; + switch (this.extensionStatusService.kind) { + case 'Error': + this.item.severity = LanguageStatusSeverity.Error; + this.item.text = `${Icon.Warning} Completions`; + this.item.detail = 'Error'; + break; + case 'Warning': + this.item.severity = LanguageStatusSeverity.Warning; + this.item.text = `${Icon.Warning} Completions`; + this.item.detail = 'Temporary issues'; + break; + case 'Inactive': + this.item.severity = LanguageStatusSeverity.Information; + this.item.text = `${Icon.Blocked} Completions`; + this.item.detail = 'Inactive'; + break; + case 'Normal': + this.item.severity = LanguageStatusSeverity.Information; + if (!isInlineSuggestEnabled()) { + this.item.text = `${Icon.NotConnected} Completions`; + this.item.detail = 'VS Code inline suggestions disabled'; + } else if (!enabled) { + this.item.text = `${Icon.NotConnected} Completions`; + this.item.detail = 'Disabled'; + } else { + this.item.text = `${Icon.Logo} Completions`; + this.item.detail = ''; + } + this.item.command.title = 'Open Menu'; + break; + } + this.item.accessibilityInformation = { + label: 'Inline Suggestions', + }; + if (this.extensionStatusService.command) { + this.item.command = this.extensionStatusService.command; + this.item.detail = this.extensionStatusService.message; + } + } + + dispose() { + for (const d of this.disposables) { + d.dispose(); + } + this.disposables = []; + } + + private isDisposed() { + return this.disposables.length === 0; + } +} diff --git a/completions-sample-code/vscode-node/extension/src/statusBarPicker.ts b/completions-sample-code/vscode-node/extension/src/statusBarPicker.ts new file mode 100644 index 0000000..5cd752d --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/statusBarPicker.ts @@ -0,0 +1,169 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { QuickPick, QuickPickItem, QuickPickItemKind, commands, l10n, window } from 'vscode'; +import { isWeb } from '../../../../../util/vs/base/common/platform'; +import { IInstantiationService } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { isCompletionEnabled, isInlineSuggestEnabled } from './config'; +import { CMDCollectDiagnosticsChat, CMDDisableCompletionsChat, CMDEnableCompletionsChat, CMDOpenDocumentationClient, CMDOpenLogsClient, CMDOpenModelPickerClient, CMDOpenPanelClient } from './constants'; +import { ICompletionsExtensionStatus } from './extensionStatus'; +import { Icon } from './icon'; + +export class CopilotStatusBarPickMenu { + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsExtensionStatus private readonly extensionStatusService: ICompletionsExtensionStatus, + ) { } + + showStatusMenu() { + const quickpickList = window.createQuickPick(); + quickpickList.placeholder = l10n.t('Select an option'); + quickpickList.title = l10n.t('Configure Inline Suggestions'); + quickpickList.items = this.collectQuickPickItems(); + quickpickList.onDidAccept(() => this.handleItemSelection(quickpickList)); + quickpickList.show(); + return quickpickList; + } + + async handleItemSelection(quickpickList: QuickPick<QuickPickItem>): Promise<void> { + const selection = quickpickList.selectedItems[0]; + if (selection === undefined) { return; } + + if ('command' in selection) { + const commandSelection = selection as CommandQuickItem; + await commands.executeCommand(commandSelection.command, ...commandSelection.commandArgs); + quickpickList.hide(); + } else { + throw new Error('Unexpected Copilot quick picker selection'); + } + } + + private collectQuickPickItems() { + return [ + this.newStatusItem(), + this.newSeparator(), + ...this.collectLanguageSpecificItems(), + this.newKeyboardItem(), + this.newSettingsItem(), + ...this.collectDiagnosticsItems(), + this.newOpenLogsItem(), + this.newSeparator(), + this.newDocsItem(), + //this.newForumItem(), + ]; + } + + private collectLanguageSpecificItems() { + const items: QuickPickItem[] = []; + if (!this.hasActiveStatus()) { return items; } + + const editor = window.activeTextEditor; + if (!isWeb && editor) { items.push(this.newPanelItem()); } + // Always show the model picker even if only one model is available + if (!isWeb) { items.push(this.newChangeModelItem()); } + if (editor) { items.push(...this.newEnableLanguageItem()); } + if (items.length) { items.push(this.newSeparator()); } + + return items; + } + + private hasActiveStatus() { + return ['Normal'].includes(this.extensionStatusService.kind); + } + + private isCompletionEnabled() { + return isInlineSuggestEnabled() && this.instantiationService.invokeFunction(isCompletionEnabled); + } + + private newEnableLanguageItem() { + const isEnabled = this.isCompletionEnabled(); + if (isEnabled) { + return [this.newCommandItem(l10n.t('Disable Inline Suggestions'), CMDDisableCompletionsChat)]; + } else if (isEnabled === false) { + return [this.newCommandItem(l10n.t('Enable Inline Suggestions'), CMDEnableCompletionsChat)]; + } else { + return []; + } + } + + private newStatusItem() { + let statusText; + let statusIcon = Icon.Logo; + switch (this.extensionStatusService.kind) { + case 'Normal': + statusText = l10n.t('Ready'); + if (isInlineSuggestEnabled() === false) { + statusText += ` (${l10n.t('VS Code inline suggestions disabled')})`; + } else if (this.instantiationService.invokeFunction(isCompletionEnabled) === false) { + statusText += ` (${l10n.t('Disabled')})`; + } + break; + case 'Inactive': + statusText = this.extensionStatusService.message || l10n.t('Copilot is currently inactive'); + statusIcon = Icon.Blocked; + break; + default: + statusText = this.extensionStatusService.message || l10n.t('Copilot has encountered an error'); + statusIcon = Icon.NotConnected; + break; + } + return this.newCommandItem(`${statusIcon} ${l10n.t('Status')}: ${statusText}`, CMDOpenLogsClient); + } + + private newOpenLogsItem() { + return this.newCommandItem(l10n.t('Open Logs...'), CMDOpenLogsClient); + } + + private collectDiagnosticsItems() { + if (isWeb) { return []; } + return [this.newCommandItem(l10n.t('Show Diagnostics...'), CMDCollectDiagnosticsChat)]; + } + + private newKeyboardItem() { + return this.newCommandItem(l10n.t('$(keyboard) Edit Keyboard Shortcuts...'), 'workbench.action.openGlobalKeybindings', [ + 'copilot', + ]); + } + + private newSettingsItem() { + return this.newCommandItem(l10n.t('$(settings-gear) Edit Settings...'), 'workbench.action.openSettings', [ + 'GitHub Copilot', + ]); + } + + private newPanelItem() { + return this.newCommandItem(l10n.t('Open Completions Panel...'), CMDOpenPanelClient); + } + + private newChangeModelItem() { + return this.newCommandItem(l10n.t('Change Completions Model...'), CMDOpenModelPickerClient); + } + + private newDocsItem() { + return this.newCommandItem( + l10n.t('$(remote-explorer-documentation) View Copilot Documentation...'), + CMDOpenDocumentationClient + ); + } + + private newCommandItem(label: string, command: string, commandArgs?: string[]): CommandQuickItem { + return new CommandQuickItem(label, command, commandArgs || []); + } + + private newSeparator(): QuickPickItem { + return { + label: '', + kind: QuickPickItemKind.Separator, + }; + } +} + +class CommandQuickItem implements QuickPickItem { + constructor( + readonly label: string, + readonly command: string, + readonly commandArgs: string[] + ) { } +} diff --git a/completions-sample-code/vscode-node/extension/src/telemetry.ts b/completions-sample-code/vscode-node/extension/src/telemetry.ts new file mode 100644 index 0000000..fb92807 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/telemetry.ts @@ -0,0 +1,45 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { commands, Disposable } from 'vscode'; +import { IDisposable } from '../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService, type ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { handleException } from '../../lib/src/defaultHandlers'; +import { Logger } from '../../lib/src/logger'; + +function exception(accessor: ServicesAccessor, error: unknown, origin: string, logger?: Logger) { + if (error instanceof Error && error.name === 'Canceled') { + // these are VS Code cancellations + return; + } + if (error instanceof Error && error.name === 'CodeExpectedError') { + // expected errors from VS Code + return; + } + handleException(accessor, error, origin, logger); +} + +export function registerCommand(accessor: ServicesAccessor, command: string, fn: (...args: unknown[]) => unknown): Disposable { + const instantiationService = accessor.get(IInstantiationService); + try { + const disposable = commands.registerCommand(command, async (...args: unknown[]) => { + try { + await fn(...args); + } catch (error) { + // Pass in the command string as the origin + instantiationService.invokeFunction(exception, error, command); + } + }); + return disposable; + } catch (error) { + console.error(`Error registering command ${command}:`, error); + throw error; + } +} + +// Wrapper that handles errors and cleans up the command on extension deactivation +export function registerCommandWrapper(accessor: ServicesAccessor, command: string, fn: (...args: unknown[]) => unknown): IDisposable { + return registerCommand(accessor, command, fn); +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/extension/src/test/config.ts b/completions-sample-code/vscode-node/extension/src/test/config.ts new file mode 100644 index 0000000..465f94e --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/test/config.ts @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ConfigKey, ConfigKeyType, DefaultsOnlyConfigProvider, InMemoryConfigProvider } from '../../../lib/src/config'; +import { VSCodeConfigProvider } from '../config'; + +/** + * Provides the default configurations, except lets through the configured value + * of test-only settings like the proxy override URL. + */ +export class ExtensionTestConfigProvider extends InMemoryConfigProvider { + private readonly vscConfigProvider = new VSCodeConfigProvider(); + + constructor() { + super(new DefaultsOnlyConfigProvider()); + } + + override getConfig<T>(key: ConfigKeyType): T { + if (key === ConfigKey.DebugTestOverrideProxyUrl) { + return this.vscConfigProvider.getConfig<T>(key); + } + return super.getConfig(key); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/test/context.ts b/completions-sample-code/vscode-node/extension/src/test/context.ts new file mode 100644 index 0000000..d3e8363 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/test/context.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { SyncDescriptor } from '../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { createExtensionTestingServices } from '../../../../../test/vscode-node/services'; +import { ICompletionsEditorAndPluginInfo } from '../../../lib/src/config'; +import { ICompletionsFileSystemService } from '../../../lib/src/fileSystem'; +import { ICompletionsFetcherService } from '../../../lib/src/networking'; +import { _createBaselineContext } from '../../../lib/src/test/context'; +import { StaticFetcher } from '../../../lib/src/test/fetcher'; +import { ICompletionsTextDocumentManagerService } from '../../../lib/src/textDocumentManager'; +import { VSCodeEditorInfo } from '../config'; +import { CopilotExtensionStatus, ICompletionsExtensionStatus } from '../extensionStatus'; +import { extensionFileSystem } from '../fileSystem'; +import { ExtensionTextDocumentManager } from '../textDocumentManager'; +import { ExtensionTestConfigProvider } from './config'; + +/** + * A default context for VSCode extension testing, building on general one in `lib`. + * Only includes items that are needed for almost all extension tests. + */ +export function createExtensionTestingContext() { + let serviceCollection = createExtensionTestingServices(); + serviceCollection = _createBaselineContext(serviceCollection, new ExtensionTestConfigProvider()); + + serviceCollection.define(ICompletionsFetcherService, new StaticFetcher()); + serviceCollection.define(ICompletionsEditorAndPluginInfo, new VSCodeEditorInfo()); + serviceCollection.define(ICompletionsTextDocumentManagerService, new SyncDescriptor(ExtensionTextDocumentManager)); + serviceCollection.define(ICompletionsFileSystemService, extensionFileSystem); + serviceCollection.define(ICompletionsExtensionStatus, new CopilotExtensionStatus()); + + return serviceCollection; +} diff --git a/completions-sample-code/vscode-node/extension/src/test/modelPicker.test.ts b/completions-sample-code/vscode-node/extension/src/test/modelPicker.test.ts new file mode 100644 index 0000000..df639f5 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/test/modelPicker.test.ts @@ -0,0 +1,113 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import sinon from 'sinon'; +import { commands, env } from 'vscode'; +import { SyncDescriptor } from '../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { AvailableModelsManager, ICompletionsModelManagerService } from '../../../lib/src/openai/model'; +import { ModelPickerManager } from './../modelPicker'; +import { createExtensionTestingContext } from './context'; + +suite('ModelPickerManager unit tests', function () { + let accessor: ServicesAccessor; + let modelPicker: ModelPickerManager; + let availableModelsManager: ICompletionsModelManagerService; + let sandbox: sinon.SinonSandbox; + + // Couple of fake models to use in our tests. + const fakeModels = [ + { + modelId: 'model-a', + label: 'Model A', + type: 'model', + alwaysShow: true, + preview: false, + tokenizer: 'o200k_base', + }, + { + modelId: 'model-b', + label: 'Model B', + type: 'model', + alwaysShow: true, + preview: false, + tokenizer: 'cl100k_base', + }, + ]; + + setup(function () { + sandbox = sinon.createSandbox(); + // Create our test context, and stub the AvailableModelsManager to return our fake models. + const serviceCollection = createExtensionTestingContext(); + serviceCollection.define(ICompletionsModelManagerService, new SyncDescriptor(AvailableModelsManager, [true])); + accessor = serviceCollection.createTestingAccessor(); + + availableModelsManager = accessor.get(ICompletionsModelManagerService); + sandbox.stub(availableModelsManager, 'getGenericCompletionModels').returns(fakeModels); + modelPicker = accessor.get(IInstantiationService).createInstance(ModelPickerManager); + }); + + teardown(async function () { + // Make sure to close any open quick pick dialogs after each test. + await commands.executeCommand('workbench.action.closeQuickOpen'); + sandbox.restore(); + }); + + test('showModelPicker returns correct items', function () { + const instantiationService = accessor.get(IInstantiationService); + + modelPicker = instantiationService.createInstance(ModelPickerManager); + + const quickPick = modelPicker.showModelPicker(); + + // Check that we have the correct number of items + // The items should include the two fake models, a separator, and a learn more item. + assert(quickPick.items.length === 4, quickPick.items.length.toString()); + assert.strictEqual(quickPick.items[0].modelId, 'model-a'); + assert.strictEqual(quickPick.items[1].modelId, 'model-b'); + assert.strictEqual(quickPick.items[2].type, 'separator'); + assert.strictEqual(quickPick.items[3].type, 'learn-more'); + }); + + test('selecting a model updates user selection', async function () { + // Stub out setting model + const setModelStub = sandbox.stub(modelPicker, 'setUserSelectedCompletionModel').resolves(); + + const quickPick = modelPicker.showModelPicker(); + + const secondItem = quickPick.items[1]; + assert(secondItem !== undefined, 'model picker should have a model-b second item.'); + + // Fake selecting the second item + quickPick.activeItems = [secondItem]; + await modelPicker.handleModelSelection(quickPick); + + // Test that we updated the user configuration with the selected model + assert(setModelStub.calledOnce, 'setUserSelectedCompletionModel should be called once'); + assert.strictEqual(setModelStub.firstCall.args[0], secondItem.modelId); + }); + + test('selecting the learn more link tries to open the learn more url', async function () { + // Stub openExternal + const openUrlStub = sandbox.stub(env, 'openExternal').resolves(); + + const quickPick = modelPicker.showModelPicker(); + + const learnMoreItem = quickPick.items[3]; + assert(learnMoreItem !== undefined, 'model picker should have a learn more item.'); + + // Fake selecting the learn more item + quickPick.activeItems = [learnMoreItem]; + await modelPicker.handleModelSelection(quickPick); + + // Test that we opened the learn more URL + assert(openUrlStub.calledOnce, 'openUrl should be called once'); + assert.strictEqual( + openUrlStub.firstCall.args[0].toString(), + 'https://aka.ms/CopilotCompletionsModelPickerLearnMore' + ); + }); +}); diff --git a/completions-sample-code/vscode-node/extension/src/textDocumentManager.ts b/completions-sample-code/vscode-node/extension/src/textDocumentManager.ts new file mode 100644 index 0000000..25e6392 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/textDocumentManager.ts @@ -0,0 +1,123 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as vscode from 'vscode'; +import { window, workspace } from 'vscode'; +import { detectLanguage } from '../../lib/src/language/languageDetection'; +import { CopilotTextDocument, INotebookCell, INotebookDocument, ITextDocument } from '../../lib/src/textDocument'; +import { TextDocumentManager, WorkspaceFoldersChangeEvent } from '../../lib/src/textDocumentManager'; +import { transformEvent } from '../../lib/src/util/event'; +import { normalizeUri } from '../../lib/src/util/uri'; + +// List of document URI schemes that avoid ghost text suggestions +const ignoreUriSchemes = new Set([ + 'output', // vscode output pane (important: avoids infinite log loop) + 'search-editor', // search results virtual document + 'comment', // very little context available and suggestions are often bad + 'git', // virtual file tracked by git + 'chat-editing-snapshot-text-model', // VS Code Chat temporary editing snapshot +]); + +export function wrapDoc(doc: vscode.TextDocument): ITextDocument | undefined { + if (ignoreUriSchemes.has(doc.uri.scheme)) { + return; + } + let text: string; + try { + text = doc.getText(); + } catch (e) { + // "Invalid string length", it's too big to fit in a string + if (e instanceof RangeError) { + return; + } + throw e; + } + const languageId = detectLanguage({ uri: doc.uri.toString(), languageId: doc.languageId }); + return CopilotTextDocument.create(doc.uri.toString(), doc.languageId, doc.version, text, languageId); +} + +export class ExtensionTextDocumentManager extends TextDocumentManager { + override onDidFocusTextDocument = transformEvent(window.onDidChangeActiveTextEditor, event => { + return { document: event && { uri: event.document.uri.toString() } }; + }); + + override onDidChangeTextDocument = transformEvent(workspace.onDidChangeTextDocument, e => { + const document = wrapDoc(e.document); + return document && { document, contentChanges: e.contentChanges }; + }); + + override onDidOpenTextDocument = transformEvent(workspace.onDidOpenTextDocument, e => { + // use wrapDoc() to handle the "Invalid string length" case + const text = wrapDoc(e)?.getText(); + if (text === undefined) { + return; + } + return { document: { uri: e.uri.toString(), languageId: e.languageId, version: e.version, text } }; + }); + + override onDidCloseTextDocument = transformEvent(workspace.onDidCloseTextDocument, e => { + return { document: { uri: normalizeUri(e.uri.toString()) } }; + }); + + override onDidChangeWorkspaceFolders = transformEvent( + workspace.onDidChangeWorkspaceFolders, + (e): WorkspaceFoldersChangeEvent => { + return { + workspaceFolders: this.getWorkspaceFolders(), + added: e.added.map(f => ({ uri: f.uri.toString(), name: f.name })), + removed: e.removed.map(f => ({ uri: f.uri.toString(), name: f.name })), + }; + } + ); + + getTextDocumentsUnsafe(): ITextDocument[] { + const docs: ITextDocument[] = []; + for (const vscodeDoc of workspace.textDocuments) { + const doc = wrapDoc(vscodeDoc); + if (doc) { + docs.push(doc); + } + } + return docs; + } + + findNotebook(doc: { uri: string }): INotebookDocument | undefined { + for (const notebook of workspace.notebookDocuments) { + if (notebook.getCells().some(cell => cell.document.uri.toString() === doc.uri.toString())) { + return { + getCells: () => notebook.getCells().map(cell => this.wrapCell(cell)), + getCellFor: ({ uri }: { uri: string }) => { + const cell = notebook.getCells().find(cell => cell.document.uri.toString() === uri.toString()); + return cell ? this.wrapCell(cell) : undefined; + }, + }; + } + } + } + + wrapCell(cell: vscode.NotebookCell): INotebookCell { + return { + ...cell, + get document(): ITextDocument { + return CopilotTextDocument.create( + cell.document.uri.toString(), + cell.document.languageId, + cell.document.version, + cell.document.getText(), + // use the original language id as cells have no metadata to leverage for language detection + cell.document.languageId + ); + }, + }; + } + + getWorkspaceFolders() { + return ( + workspace.workspaceFolders?.map(f => { + return { uri: f.uri.toString(), name: f.name }; + }) ?? [] + ); + } +} diff --git a/completions-sample-code/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts b/completions-sample-code/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts new file mode 100644 index 0000000..feff170 --- /dev/null +++ b/completions-sample-code/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts @@ -0,0 +1,210 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + CancellationToken, + InlineCompletionContext, + InlineCompletionEndOfLifeReason, + InlineCompletionItemProvider, + InlineCompletionList, + InlineCompletionTriggerKind, + PartialAcceptInfo, + Position, + TextDocument, + workspace +} from 'vscode'; +import { Disposable } from '../../../../../util/vs/base/common/lifecycle'; +import { LineEdit } from '../../../../../util/vs/editor/common/core/edits/lineEdit'; +import { TextEdit, TextReplacement } from '../../../../../util/vs/editor/common/core/edits/textEdit'; +import { Range } from '../../../../../util/vs/editor/common/core/range'; +import { LineBasedText } from '../../../../../util/vs/editor/common/core/text/abstractText'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { InlineEditLogger } from '../../../../inlineEdits/vscode-node/parts/inlineEditLogger'; +import { GhostTextContext } from '../../../common/ghostTextContext'; +import { ICompletionsTelemetryService } from '../../bridge/src/completionsTelemetryServiceBridge'; +import { BuildInfo } from '../../lib/src/config'; +import { CopilotConfigPrefix } from '../../lib/src/constants'; +import { handleException } from '../../lib/src/defaultHandlers'; +import { Logger } from '../../lib/src/logger'; +import { isCompletionEnabledForDocument } from './config'; +import { CopilotCompletionFeedbackTracker, sendCompletionFeedbackCommand } from './copilotCompletionFeedbackTracker'; +import { ICompletionsExtensionStatus } from './extensionStatus'; +import { GhostTextCompletionItem, GhostTextCompletionList, GhostTextProvider } from './ghostText/ghostTextProvider'; + +const logger = new Logger('inlineCompletionItemProvider'); + +function quickSuggestionsDisabled() { + const qs = workspace.getConfiguration('editor.quickSuggestions'); + return qs.get('other') !== 'on' && qs.get('comments') !== 'on' && qs.get('strings') !== 'on'; +} + +export function exception(accessor: ServicesAccessor, error: unknown, origin: string, logger?: Logger) { + if (error instanceof Error && error.name === 'Canceled') { + // these are VS Code cancellations + return; + } + if (error instanceof Error && error.name === 'CodeExpectedError') { + // expected errors from VS Code + return; + } + const telemetryService = accessor.get(ICompletionsTelemetryService); + telemetryService.sendGHTelemetryException(error, 'codeUnification.completions.exception'); + handleException(accessor, error, origin, logger); +} + +/** @public */ +export class CopilotInlineCompletionItemProvider extends Disposable implements InlineCompletionItemProvider { + private readonly copilotCompletionFeedbackTracker: CopilotCompletionFeedbackTracker; + private readonly ghostTextProvider: GhostTextProvider; + private readonly inlineEditLogger: InlineEditLogger; + + public onDidChange = undefined; + public handleListEndOfLifetime: InlineCompletionItemProvider['handleListEndOfLifetime'] = undefined; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsTelemetryService private readonly telemetryService: ICompletionsTelemetryService, + @ICompletionsExtensionStatus private readonly extensionStatusService: ICompletionsExtensionStatus, + ) { + super(); + this.copilotCompletionFeedbackTracker = this._register(this.instantiationService.createInstance(CopilotCompletionFeedbackTracker)); + this.ghostTextProvider = this.instantiationService.createInstance(GhostTextProvider); + this.inlineEditLogger = this.instantiationService.createInstance(InlineEditLogger); + } + + async provideInlineCompletionItems( + doc: TextDocument, + position: Position, + context: InlineCompletionContext, + token: CancellationToken + ): Promise<GhostTextCompletionList | undefined> { + const logContext = new GhostTextContext(doc.uri.toString(), doc.version, context); + try { + return await this._provideInlineCompletionItems(doc, position, context, logContext, token); + } catch (e) { + logContext.setError(e); + this.telemetryService.sendGHTelemetryException(e, 'codeUnification.completions.exception'); + } finally { + this.inlineEditLogger.add(logContext); + } + } + + private async _provideInlineCompletionItems( + doc: TextDocument, + position: Position, + context: InlineCompletionContext, + logContext: GhostTextContext, + token: CancellationToken + ): Promise<GhostTextCompletionList | undefined> { + if (context.triggerKind === InlineCompletionTriggerKind.Automatic) { + if (!this.instantiationService.invokeFunction(isCompletionEnabledForDocument, doc)) { + return; + } + if (this.extensionStatusService.kind === 'Error') { + return; + } + } + const copilotConfig = workspace.getConfiguration(CopilotConfigPrefix); + // Constraining the generated inline completion to match selectedCompletionInfo sandbags Copilot pretty hard, as + // typically it's just the first entry in the list alphabetically. But if we generate a result that doesn't + // match it, VS Code won't show it to the user unless the completion dropdown is dismissed. Historically we've + // chosen to favor completion quality, but this option allows opting into or out of generating a completion that + // VS Code will actually show. + if (!copilotConfig.get('respectSelectedCompletionInfo', quickSuggestionsDisabled() || BuildInfo.isPreRelease())) { + context = { ...context, selectedCompletionInfo: undefined }; + } + + try { + let items = await this.ghostTextProvider.provideInlineCompletionItems(doc, position, context, token); + + if (!items) { + if (token.isCancellationRequested) { + logContext.setIsSkipped(); + } + return undefined; + } + + // If the language client provides a list of items, we want to add the send feedback command to it. + if (Array.isArray(items)) { + items = { items }; + } + + this.logSuggestion(logContext, doc, items); + + return { + ...items, + commands: [sendCompletionFeedbackCommand], + }; + } catch (e) { + this.instantiationService.invokeFunction(exception, e, '._provideInlineCompletionItems', logger); + logContext.setError(e); + } + } + + handleDidShowCompletionItem(item: GhostTextCompletionItem) { + try { + this.copilotCompletionFeedbackTracker.trackItem(item); + return this.ghostTextProvider.handleDidShowCompletionItem(item); + } catch (e) { + this.instantiationService.invokeFunction(exception, e, '.handleDidShowCompletionItem', logger); + } + } + + handleDidPartiallyAcceptCompletionItem( + item: GhostTextCompletionItem, + acceptedLengthOrInfo: number | PartialAcceptInfo + ) { + try { + return this.ghostTextProvider.handleDidPartiallyAcceptCompletionItem(item, acceptedLengthOrInfo); + } catch (e) { + this.instantiationService.invokeFunction(exception, e, '.handleDidPartiallyAcceptCompletionItem', logger); + } + } + + handleEndOfLifetime(completionItem: GhostTextCompletionItem, reason: InlineCompletionEndOfLifeReason) { + try { + return this.ghostTextProvider.handleEndOfLifetime(completionItem, reason); + } catch (e) { + this.instantiationService.invokeFunction(exception, e, '.handleEndOfLifetime', logger); + } + } + + private logSuggestion( + logContext: GhostTextContext, + doc: TextDocument, + items: InlineCompletionList + ) { + if (items.items.length === 0) { + logContext.markAsNoSuggestions(); + logContext.addLog('No inline completion items provided'); + return; + } + const firstItem = items.items[0]; + if (!firstItem.range) { + logContext.addLog('Inline completion item has no range'); + return; + } + if (typeof firstItem.insertText !== 'string') { + logContext.addLog('Inline completion item has non-string insertText'); + return; + } + + const text = new LineBasedText(lineNumber => doc.lineAt(lineNumber - 1).text, doc.lineCount); + + const lineEdit = LineEdit.fromTextEdit( + new TextEdit( + [new TextReplacement( + new Range(firstItem.range.start.line + 1, firstItem.range.start.character + 1, firstItem.range.end.line + 1, firstItem.range.end.character + 1), + firstItem.insertText, + )], + ), + text + ); + + const patch = lineEdit.humanReadablePatch(text.getLines()); + + logContext.setResult(patch); + } +} diff --git a/completions-sample-code/vscode-node/extension/test/run.js b/completions-sample-code/vscode-node/extension/test/run.js new file mode 100644 index 0000000..925747b --- /dev/null +++ b/completions-sample-code/vscode-node/extension/test/run.js @@ -0,0 +1,75 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +require('tsx/cjs'); + +const { globSync } = require('glob'); +const Mocha = require('mocha'); +const path = require('path'); +const dotenv = require('dotenv'); +const envfile = path.join(__dirname, '../../../../../../.env'); +dotenv.config({ path: envfile }); + +function run() { + const projectRoot = path.resolve(__dirname, '../..'); + const mochaOptions = { + ui: 'tdd', + color: !process.env.NO_COLOR && process.env.TERM !== 'dumb', + reporter: 'mocha-multi-reporters', + reporterOptions: { + reporterEnabled: 'spec', + }, + }; + if (process.env.MOCHA_GREP) { + mochaOptions.grep = process.env.MOCHA_GREP; + } + if (process.env.CI) { + mochaOptions.forbidOnly = true; + mochaOptions.retries = 2; + mochaOptions.reporterOptions.reporterEnabled += ', mocha-junit-reporter'; + mochaOptions.reporterOptions.mochaJunitReporterReporterOptions = { + testCaseSwitchClassnameAndName: true, + testsuitesTitle: 'Copilot VS Code Extension Tests', + mochaFile: path.resolve(projectRoot, 'test-results-Extension.xml'), + }; + } + if (process.env.GITHUB_EVENT_NAME === 'merge_group') { + mochaOptions.retries = 3; + } + + // Create the mocha test + const mocha = new Mocha(mochaOptions); + + let fileCount = 0; + (process.env.MOCHA_FILES || [ + path.resolve(projectRoot, 'lib/src/**/*.test.{ts,tsx}'), + path.resolve(projectRoot, 'extension/src/**/*.test.{ts,tsx}') + ].join('\n')).split('\n').forEach(f => { + globSync(f, { windowsPathsNoEscape: true }).forEach(f => { + fileCount++; + mocha.addFile(f); + }); + }); + if (!fileCount) { + throw new Error('No tests to run'); + } + + return new Promise((c, e) => { + try { + // Run the mocha test + mocha.run(failures => { + if (failures > 0) { + e(new Error(`${failures} tests failed.`)); + } else { + c(); + } + }); + } catch (err) { + e(err); + } + }); +} + +module.exports = { run }; diff --git a/completions-sample-code/vscode-node/extension/test/runTest.ts b/completions-sample-code/vscode-node/extension/test/runTest.ts new file mode 100644 index 0000000..708f9fc --- /dev/null +++ b/completions-sample-code/vscode-node/extension/test/runTest.ts @@ -0,0 +1,79 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; + +import { runTests } from '@vscode/test-electron'; + +async function main() { + const tempdir = await fs.mkdtemp(os.tmpdir() + '/copilot-extension-test-'); + + let exitCode; + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionDevelopmentPath = path.resolve(__dirname, '../..'); + + // The path to the extension test script (must be javascript) + // Passed to --extensionTestsPath + const extensionTestsPath = path.resolve(__dirname, './run'); + + const launchArgs = []; + // Disable other extensions while testing, + launchArgs.push('--disable-extensions'); + + // use a temporary folder so we can run multiple instances of the same VS Code together + // see https://github.com/microsoft/vscode/issues/137678 + launchArgs.push('--user-data-dir', tempdir); + + const argv = await yargs(hideBin(process.argv)) + .options({ + stable: { + type: 'boolean', + default: false, + }, + grep: { + alias: 'g', + type: 'string', + default: '', + }, + }) + .parse(); + const version = argv.stable ? 'stable' : 'insiders'; + + const extensionTestsEnv: typeof process.env = {}; + // Pass arguments to mocha by environment variables + if (argv.grep) { extensionTestsEnv.MOCHA_GREP = argv.grep; } + if (argv._.length > 0) { extensionTestsEnv.MOCHA_FILES = argv._.join('\n'); } + if (!process.stdout.isTTY) { extensionTestsEnv.NO_COLOR = 'true'; } + const workspaceFolder = await fs.mkdtemp(path.join(os.tmpdir(), 'copilot-extension-test-')); + launchArgs.push(workspaceFolder); + + extensionTestsEnv.CORETEST = 'true'; + //@dbaeumer This can be removed as soon as we have the cache handle CORETEST + extensionTestsEnv.VITEST = 'true'; + + // Download VS Code, unzip it and run the integration test + exitCode = await runTests({ + version, + extensionDevelopmentPath, + extensionTestsPath, + launchArgs, + extensionTestsEnv, + }); + } catch (err) { + console.error('Failed to run tests', err); + exitCode = 1; + } finally { + await fs.rm(tempdir, { recursive: true }); + } + process.exit(exitCode); +} + +void main(); diff --git a/completions-sample-code/vscode-node/lib/src/auth/copilotTokenManager.ts b/completions-sample-code/vscode-node/lib/src/auth/copilotTokenManager.ts new file mode 100644 index 0000000..896b5c3 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/auth/copilotTokenManager.ts @@ -0,0 +1,72 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { CopilotToken } from '../../../../../../platform/authentication/common/copilotToken'; +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { ThrottledDelayer } from '../../../../../../util/vs/base/common/async'; +import { Disposable } from '../../../../../../util/vs/base/common/lifecycle'; +export { CopilotToken } from '../../../../../../platform/authentication/common/copilotToken'; + +export const ICompletionsCopilotTokenManager = createServiceIdentifier<ICompletionsCopilotTokenManager>('ICompletionsCopilotTokenManager'); +export interface ICompletionsCopilotTokenManager { + readonly _serviceBrand: undefined; + get token(): CopilotToken | undefined; + primeToken(): Promise<boolean>; + getToken(): Promise<CopilotToken>; + resetToken(httpError?: number): void; + getLastToken(): Omit<CopilotToken, 'token'> | undefined; +} + +export class CopilotTokenManagerImpl extends Disposable implements ICompletionsCopilotTokenManager { + declare _serviceBrand: undefined; + private tokenRefetcher = new ThrottledDelayer(5_000); + private _token: CopilotToken | undefined; + get token() { + void this.tokenRefetcher.trigger(() => this.updateCachedToken()); + return this._token; + } + + constructor( + protected primed = false, + @IAuthenticationService private readonly authenticationService: IAuthenticationService + ) { + super(); + + this.updateCachedToken(); + this._register(this.authenticationService.onDidAuthenticationChange(() => this.updateCachedToken())); + } + + /** + * Ensure we have a token and that the `StatusReporter` is up to date. + */ + primeToken(): Promise<boolean> { + try { + return this.getToken().then( + () => true, + () => false + ); + } catch (e) { + return Promise.resolve(false); + } + } + + async getToken(): Promise<CopilotToken> { + return this.updateCachedToken(); + } + + private async updateCachedToken(): Promise<CopilotToken> { + this._token = await this.authenticationService.getCopilotToken(); + return this._token; + } + + resetToken(httpError?: number): void { + this.authenticationService.resetCopilotToken(); + } + + getLastToken(): Omit<CopilotToken, 'token'> | undefined { + return this.authenticationService.copilotToken; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/auth/copilotTokenNotifier.ts b/completions-sample-code/vscode-node/lib/src/auth/copilotTokenNotifier.ts new file mode 100644 index 0000000..b0b6765 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/auth/copilotTokenNotifier.ts @@ -0,0 +1,16 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { CopilotToken } from '../../../../../../platform/authentication/common/copilotToken'; + +export function onCopilotToken(authService: IAuthenticationService, listener: (token: Omit<CopilotToken, 'token'>) => unknown) { + return authService.onDidAuthenticationChange(() => { + const copilotToken = authService.copilotToken; + if (copilotToken) { + listener(copilotToken); + } + }); +} diff --git a/completions-sample-code/vscode-node/lib/src/auth/orgs.ts b/completions-sample-code/vscode-node/lib/src/auth/orgs.ts new file mode 100644 index 0000000..0eeac87 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/auth/orgs.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotToken } from './copilotTokenManager'; + +/** + * A function used to determine if the org list contains an known organization + * @param orgs The list of organizations the user is a member of + * @returns The first known organization or undefined if none are known. + */ +function findKnownOrg(orgs: string[]): string | undefined { + // Do not add org mapping + const known_orgs = [ + 'a5db0bcaae94032fe715fb34a5e4bce2', + '7184f66dfcee98cb5f08a1cb936d5225', + 'faef89d9169d5eacf1d8c8dde3412e37', + '4535c7beffc844b46bb1ed4aa04d759a', + ]; + return known_orgs.find(o => orgs.includes(o)); +} + +export function getUserKind(token: Omit<CopilotToken, 'token'>): string { + const orgs = token.organizationList ?? []; + return findKnownOrg(orgs) ?? ''; +} diff --git a/completions-sample-code/vscode-node/lib/src/changeTracker.ts b/completions-sample-code/vscode-node/lib/src/changeTracker.ts new file mode 100644 index 0000000..d8645f8 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/changeTracker.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { Disposable } from '../../types/src'; +import { ICompletionsTextDocumentManagerService } from './textDocumentManager'; + +/** + * A tracker which can take an arbitrary number of actions to run after a given timeout + * When all pushed timeouts have been resolved, the tracker disposes of itself. + */ +export class ChangeTracker { + private _offset: number; + get offset(): number { + return this._offset; + } + private _referenceCount = 0; + private _tracker: Disposable; + private _isDisposed = false; + + constructor( + fileURI: string, + insertionOffset: number, + @ICompletionsTextDocumentManagerService documentManager: ICompletionsTextDocumentManagerService + ) { + this._offset = insertionOffset; + + this._tracker = documentManager.onDidChangeTextDocument(e => { + if (e.document.uri === fileURI) { + for (const cc of e.contentChanges) { + if (cc.rangeOffset + cc.rangeLength <= this.offset) { + const delta = cc.text.length - cc.rangeLength; + this._offset = this._offset + delta; + } + } + } + }); + } + + push(action: () => void, timeout: number): void { + if (this._isDisposed) { + throw new Error('Unable to push new actions to a disposed ChangeTracker'); + } + this._referenceCount++; + setTimeout(() => { + action(); + this._referenceCount--; + if (this._referenceCount === 0) { + this._tracker.dispose(); + this._isDisposed = true; + } + }, timeout); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/citationManager.ts b/completions-sample-code/vscode-node/lib/src/citationManager.ts new file mode 100644 index 0000000..b9537b1 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/citationManager.ts @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { Disposable, IDisposable } from '../../../../../util/vs/base/common/lifecycle'; +import { IRange } from './textDocument'; + +export interface IPCitationDetail { + license: string; + url: string; +} + +export interface IPDocumentCitation { + inDocumentUri: string; + offsetStart: number; + offsetEnd: number; + version?: number; + location?: IRange; + matchingText?: string; + details: IPCitationDetail[]; +} + +export const ICompletionsCitationManager = createServiceIdentifier<ICompletionsCitationManager>('ICompletionsCitationManager'); +export interface ICompletionsCitationManager { + readonly _serviceBrand: undefined; + + register(): IDisposable; + handleIPCodeCitation(citation: IPDocumentCitation): Promise<void>; +} + +export class NoOpCitationManager implements ICompletionsCitationManager { + declare _serviceBrand: undefined; + + register() { return Disposable.None; } + + async handleIPCodeCitation(citation: IPDocumentCitation): Promise<void> { + // Do nothing + } +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/completionNotifier.ts b/completions-sample-code/vscode-node/lib/src/completionNotifier.ts new file mode 100644 index 0000000..b7555d3 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/completionNotifier.ts @@ -0,0 +1,67 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import EventEmitter from 'events'; +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { ICompletionsTelemetryService } from '../../bridge/src/completionsTelemetryServiceBridge'; +import { CancellationToken, Disposable } from '../../types/src'; +import { CompletionState } from './completionState'; +import { GetGhostTextOptions } from './ghostText/ghostText'; +import { telemetryCatch, TelemetryWithExp } from './telemetry'; +import { ICompletionsPromiseQueueService } from './util/promiseQueue'; + +export type CompletionRequestedEvent = { + completionId: string; + completionState: CompletionState; + telemetryData: TelemetryWithExp; + cancellationToken?: CancellationToken; + options?: Partial<GetGhostTextOptions>; +}; + +const requestEventName = 'CompletionRequested'; + +export const ICompletionsNotifierService = createServiceIdentifier<ICompletionsNotifierService>('ICompletionsNotifierService'); +export interface ICompletionsNotifierService { + readonly _serviceBrand: undefined; + notifyRequest( + completionState: CompletionState, + completionId: string, + telemetryData: TelemetryWithExp, + cancellationToken?: CancellationToken, + options?: Partial<GetGhostTextOptions> + ): void; + + onRequest(listener: (event: CompletionRequestedEvent) => void): Disposable; +} + +export class CompletionNotifier implements ICompletionsNotifierService { + declare _serviceBrand: undefined; + #emitter = new EventEmitter(); + constructor( + @ICompletionsPromiseQueueService protected completionsPromiseQueue: ICompletionsPromiseQueueService, + @ICompletionsTelemetryService protected completionsTelemetryService: ICompletionsTelemetryService, + ) { } + + notifyRequest( + completionState: CompletionState, + completionId: string, + telemetryData: TelemetryWithExp, + cancellationToken?: CancellationToken, + options?: Partial<GetGhostTextOptions> + ) { + return this.#emitter.emit(requestEventName, { + completionId, + completionState, + telemetryData, + cancellationToken, + options, + }); + } + + onRequest(listener: (event: CompletionRequestedEvent) => void): Disposable { + const wrapper = telemetryCatch(this.completionsTelemetryService, this.completionsPromiseQueue, listener, `event.${requestEventName}`); + this.#emitter.on(requestEventName, wrapper); + return Disposable.create(() => this.#emitter.off(requestEventName, wrapper)); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/completionState.ts b/completions-sample-code/vscode-node/lib/src/completionState.ts new file mode 100644 index 0000000..77cb334 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/completionState.ts @@ -0,0 +1,118 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { Position, ProposedTextEdit, TextEdit } from '../../types/src'; +import { IntelliSenseInsertion, ITextDocument, type TextDocumentContents } from './textDocument'; + +export class CompletionState { + readonly originalPosition: Position; + readonly originalVersion: number; + readonly originalOffset: number; + private readonly _editsWithPosition: ReadonlyArray<ProposedTextEdit>; + + constructor( + private readonly _textDocument: ITextDocument, + private readonly _position: Position, + edits: ProposedTextEdit[] = [], + originalPosition?: Position, + originalVersion?: number, + originalOffset?: number + ) { + this.originalPosition = originalPosition ?? Position.create(_position.line, _position.character); + this.originalVersion = originalVersion ?? _textDocument.version; + this.originalOffset = originalOffset ?? _textDocument.offsetAt(this.originalPosition); + this._editsWithPosition = [...edits]; + } + + get textDocument(): TextDocumentContents { + return this._textDocument; + } + + get position(): Position { + return this._position; + } + + get editsWithPosition(): ProposedTextEdit[] { + return [...this._editsWithPosition]; + } + + private updateState(textDocument: ITextDocument, position: Position, edits?: ProposedTextEdit[]): CompletionState { + return new CompletionState( + textDocument, + position, + edits ?? this.editsWithPosition, + this.originalPosition, + this.originalVersion, + this.originalOffset + ); + } + + updatePosition(position: Position): CompletionState { + return this.updateState(this._textDocument, position); + } + + addSelectedCompletionInfo(selectedCompletionInfo: IntelliSenseInsertion): CompletionState { + if (this.editsWithPosition.find(edit => edit.source === 'selectedCompletionInfo')) { + throw new Error('Selected completion info already applied'); + } + + const edit: TextEdit = { + range: selectedCompletionInfo.range, + newText: selectedCompletionInfo.text, + }; + return this.applyEdits([edit], true); + } + + applyEdits(edits: TextEdit[], isSelectedCompletionInfo = false): CompletionState { + if (isSelectedCompletionInfo && edits.length > 1) { + throw new Error('Selected completion info should be a single edit'); + } + + let textDocument = this._textDocument; + let position = this._position; + let offset: number = textDocument.offsetAt(position); + const newEdits = this.editsWithPosition; + + for (const { range, newText } of edits) { + const oldText = textDocument.getText(range); + const oldEndOffset = textDocument.offsetAt(range.end); + textDocument = textDocument.applyEdits([{ range, newText }]); + // We err on the side of updating the position if it's exactly aligned with the start of the range. This is + // what we want in the context of applying a completion, but it does make some operations impossible, like + // preserving a position at the start of the document (line 0 column 0). + if (offset < textDocument.offsetAt(range.start)) { + const edit: ProposedTextEdit = { + range, + newText, + positionAfterEdit: Position.create(position.line, position.character), + }; + if (isSelectedCompletionInfo) { + edit.source = 'selectedCompletionInfo'; + } + newEdits.push(edit); + continue; + } + if (offset < oldEndOffset) { + offset = oldEndOffset; + } + offset += newText.length - oldText.length; + position = textDocument.positionAt(offset); + const edit: ProposedTextEdit = { + range, + newText, + positionAfterEdit: Position.create(position.line, position.character), + }; + if (isSelectedCompletionInfo) { + edit.source = 'selectedCompletionInfo'; + } + newEdits.push(edit); + } + + return this.updateState(textDocument, position, newEdits); + } +} + +export function createCompletionState(textDocument: ITextDocument, position: Position): CompletionState { + return new CompletionState(textDocument, position); +} diff --git a/completions-sample-code/vscode-node/lib/src/completionsObservableWorkspace.ts b/completions-sample-code/vscode-node/lib/src/completionsObservableWorkspace.ts new file mode 100644 index 0000000..cd23b6e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/completionsObservableWorkspace.ts @@ -0,0 +1,22 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { DocumentId } from '../../../../../platform/inlineEdits/common/dataTypes/documentId'; +import { IObservableDocument } from '../../../../../platform/inlineEdits/common/observableWorkspace'; +import { IObservableWithChange } from '../../../../../util/vs/base/common/observableInternal'; +import { URI } from '../../../../../util/vs/base/common/uri'; +import { createDecorator as createServiceIdentifier } from '../../../../../util/vs/platform/instantiation/common/instantiation'; + +export const ICompletionsObservableWorkspace = createServiceIdentifier<ICompletionsObservableWorkspace>('ICompletionsObservableWorkspace'); +export interface ICompletionsObservableWorkspace { + readonly _serviceBrand: undefined; + + get openDocuments(): IObservableWithChange<readonly IObservableDocument[], { added: readonly IObservableDocument[]; removed: readonly IObservableDocument[] }>; + + getWorkspaceRoot(documentId: DocumentId): URI | undefined; + + getFirstOpenDocument(): IObservableDocument | undefined; + + getDocument(documentId: DocumentId): IObservableDocument | undefined; +} diff --git a/completions-sample-code/vscode-node/lib/src/config.ts b/completions-sample-code/vscode-node/lib/src/config.ts new file mode 100644 index 0000000..93e4f5c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/config.ts @@ -0,0 +1,394 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { packageJson } from '../../../../../platform/env/common/packagejson'; +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotConfigPrefix } from './constants'; +import { Filter } from './experiments/filters'; +import { Emitter, Event } from './util/event'; + +export { packageJson }; + +export const ConfigKey = { + Enable: 'enable', + UserSelectedCompletionModel: 'selectedCompletionModel', + + ShowEditorCompletions: 'editor.showEditorCompletions', + EnableAutoCompletions: 'editor.enableAutoCompletions', + DelayCompletions: 'editor.delayCompletions', + FilterCompletions: 'editor.filterCompletions', + CompletionsDelay: 'completionsDelay', + CompletionsDebounce: 'completionsDebounce', + + // Advanced config (don't add new config here) + RelatedFilesVSCodeCSharp: 'advanced.relatedFilesVSCodeCSharp', + RelatedFilesVSCodeTypeScript: 'advanced.relatedFilesVSCodeTypeScript', + RelatedFilesVSCode: 'advanced.relatedFilesVSCode', + ContextProviders: 'advanced.contextProviders', + DebugFilterLogCategories: 'advanced.debug.filterLogCategories', + DebugSnippyOverrideUrl: 'advanced.debug.codeRefOverrideUrl', + UseSubsetMatching: 'advanced.useSubsetMatching', + ContextProviderTimeBudget: 'advanced.contextProviderTimeBudget', + + // Internal config + DebugOverrideCapiUrl: 'internal.capiUrl', + DebugOverrideCapiUrlLegacy: 'advanced.debug.overrideCapiUrl', + DebugTestOverrideCapiUrl: 'internal.capiTestUrl', + DebugTestOverrideCapiUrlLegacy: 'advanced.debug.testOverrideCapiUrl', + DebugOverrideProxyUrl: 'internal.completionsUrl', + DebugOverrideProxyUrlLegacy: 'advanced.debug.overrideProxyUrl', + DebugTestOverrideProxyUrl: 'internal.completionsTestUrl', + DebugTestOverrideProxyUrlLegacy: 'advanced.debug.testOverrideProxyUrl', + DebugOverrideEngine: 'internal.completionModel', + DebugOverrideEngineLegacy: 'advanced.debug.overrideEngine', + /** + * Internal experiment for always requesting multiline completions. + * This might not result always in a multiline suggestion, but most often will. + */ + AlwaysRequestMultiline: 'internal.alwaysRequestMultiline', + /** + * Let the model terminate single line completions when AlwaysRequestMultiline is enabled. + */ + ModelAlwaysTerminatesSingleline: 'internal.modelAlwaysTerminatesSingleline', + + /** + * Overrides whether to use the Workspace Context Coordinator to coordinate workspace context. + * This setting takes precedence over the value from ExP. + */ + UseWorkspaceContextCoordinator: 'internal.useWorkspaceContextCoordinator', + + /** + * Overrides whether to include neighboring files in the prompt + * alongside context providers. + * This setting takes precedence over the value from ExP. + */ + IncludeNeighboringFiles: 'internal.includeNeighboringFiles', + ExcludeRelatedFiles: 'internal.excludeRelatedFiles', + DebugOverrideCppHeadersEnableSwitch: 'internal.cppHeadersEnableSwitch', + + /** + * Internal config for using the completions prompt with split context. + * https://github.com/github/copilot/issues/19286 + */ + UseSplitContextPrompt: 'internal.useSplitContextPrompt', +}; + +export type ConfigKeyType = string; + +// How to determine where to terminate the completion to the current block. +export enum BlockMode { + /** + * Parse the context + completion on the client using treesitter to + * determine blocks. + */ + Parsing = 'parsing', + /** + * Let the server parse out blocks and assume that the completion terminates + * at the end of a block. + */ + Server = 'server', + /** + * Runs both the treesitter parsing on the client plus indentation-based + * truncation on the proxy. + */ + ParsingAndServer = 'parsingandserver', + /** + * Client-based heuristic to display more multiline completions. + * It almost always requests a multiline completion from the server and tries to break it up to something useful on the client. + * + * This should not be rolled out at the moment (latency impact is high, UX needs further fine-tuning), + * but can be used for internal experimentation. + */ + MoreMultiline = 'moremultiline', +} + +export function shouldDoServerTrimming(blockMode: BlockMode): boolean { + return [BlockMode.Server, BlockMode.ParsingAndServer].includes(blockMode); +} + +// TODO rework this enum so that the normal/nightly and prod/dev distinctions are orthogonal. (dev builds should behave like nightly?) +export enum BuildType { + DEV = 'dev', + PROD = 'prod', + NIGHTLY = 'nightly', +} + +export const ICompletionsConfigProvider = createServiceIdentifier<ICompletionsConfigProvider>('ICompletionsConfigProvider'); +export interface ICompletionsConfigProvider { + readonly _serviceBrand: undefined; + + getConfig<T>(key: ConfigKeyType): T; + getOptionalConfig<T>(key: ConfigKeyType): T | undefined; + dumpForTelemetry(): { [key: string]: string }; + onDidChangeCopilotSettings: Event<ConfigProvider>; +} + +export abstract class ConfigProvider implements ICompletionsConfigProvider { + declare _serviceBrand: undefined; + abstract getConfig<T>(key: ConfigKeyType): T; + abstract getOptionalConfig<T>(key: ConfigKeyType): T | undefined; + abstract dumpForTelemetry(): { [key: string]: string }; + abstract onDidChangeCopilotSettings: Event<ConfigProvider>; + + // The language server receives workspace configuration *after* it is fully initialized, which creates a race + // condition where an incoming request immediately after initialization might have the default values. Awaiting + // this promise allows consumers to ensure that the configuration is ready before using it. + requireReady(): Promise<void> { + return Promise.resolve(); + } +} + +/** Provides only the default values, ignoring the user's settings. + * @public KEEPING FOR TESTS +*/ +export class DefaultsOnlyConfigProvider extends ConfigProvider { + override getConfig<T>(key: ConfigKeyType): T { + // hardcode default values for the agent, for now + return getConfigDefaultForKey<T>(key); + } + + override getOptionalConfig<T>(key: ConfigKeyType): T | undefined { + return getOptionalConfigDefaultForKey<T>(key); + } + + override dumpForTelemetry(): { [key: string]: string } { + return {}; + } + + override onDidChangeCopilotSettings = () => { + // no-op, since this provider does not support changing settings + return { + dispose: () => { }, + }; + }; +} + +/** + * A ConfigProvider that allows overriding of config values. + * @public KEEPING FOR TESTS +*/ +export class InMemoryConfigProvider extends ConfigProvider { + protected readonly copilotEmitter = new Emitter<this>(); + readonly onDidChangeCopilotSettings = this.copilotEmitter.event; + private overrides: Map<ConfigKeyType, unknown> = new Map(); + + constructor( + private readonly baseConfigProvider: ConfigProvider, + ) { + super(); + } + + setOverrides(overrides: Map<ConfigKeyType, unknown>): void { + this.overrides = overrides; + } + + clearOverrides(): void { + this.overrides.clear(); + } + + protected getOptionalOverride<T>(key: ConfigKeyType): T | undefined { + return this.overrides.get(key) as T | undefined; + } + + override getConfig<T>(key: ConfigKeyType): T { + return this.getOptionalOverride(key) ?? this.baseConfigProvider.getConfig(key); + } + + override getOptionalConfig<T>(key: ConfigKeyType): T | undefined { + return this.getOptionalOverride(key) ?? this.baseConfigProvider.getOptionalConfig(key); + } + + setConfig(key: ConfigKeyType, value: unknown): void { + this.setCopilotSettings({ [key]: value }); + } + + setCopilotSettings(settings: Record<ConfigKeyType, unknown>): void { + for (const [key, value] of Object.entries(settings)) { + if (value !== undefined) { + this.overrides.set(key, value); + } else { + this.overrides.delete(key); + } + } + this.copilotEmitter.fire(this); + } + + override dumpForTelemetry(): { [key: string]: string } { + const config = this.baseConfigProvider.dumpForTelemetry(); + // reflects what's mapped in Hydro + for (const key of [ + ConfigKey.ShowEditorCompletions, + ConfigKey.EnableAutoCompletions, + ConfigKey.DelayCompletions, + ConfigKey.FilterCompletions, + ]) { + const value = this.overrides.get(key); + if (value !== undefined) { + config[key] = JSON.stringify(value); + } + } + return config; + } + + +} + +export function getConfigKeyRecursively<T>(config: Record<string, unknown>, key: string): T | undefined { + let value: unknown = config; + const prefix: string[] = []; + for (const segment of key.split('.')) { + const child = [...prefix, segment].join('.'); + if (value && typeof value === 'object' && child in value) { + value = (value as { [key: string]: unknown })[child]; + prefix.length = 0; + } else { + prefix.push(segment); + } + } + if (value === undefined || prefix.length > 0) { return; } + return value as T; +} + +export function getConfigDefaultForKey<T>(key: string): T { + if (configDefaults.has(key)) { + return configDefaults.get(key) as T; + } + throw new Error(`Missing config default value: ${CopilotConfigPrefix}.${key}`); +} + +export function getOptionalConfigDefaultForKey<T>(key: string): T | undefined { + return <T>configDefaults.get(key); +} + +/** + * Defaults for "hidden" config keys. These are supplemented by the defaults in package.json. + */ +const configDefaults = new Map<ConfigKeyType, unknown>([ + [ConfigKey.DebugOverrideCppHeadersEnableSwitch, false], + [ConfigKey.RelatedFilesVSCodeCSharp, false], + [ConfigKey.RelatedFilesVSCodeTypeScript, false], + [ConfigKey.RelatedFilesVSCode, false], + [ConfigKey.IncludeNeighboringFiles, false], + [ConfigKey.ExcludeRelatedFiles, false], + [ConfigKey.ContextProviders, []], + [ConfigKey.DebugSnippyOverrideUrl, ''], + [ConfigKey.UseSubsetMatching, null], + [ConfigKey.ContextProviderTimeBudget, undefined], + [ConfigKey.DebugOverrideCapiUrl, ''], + [ConfigKey.DebugTestOverrideCapiUrl, ''], + [ConfigKey.DebugOverrideProxyUrl, ''], + [ConfigKey.DebugTestOverrideProxyUrl, ''], + [ConfigKey.DebugOverrideEngine, ''], + [ConfigKey.AlwaysRequestMultiline, undefined], + [ConfigKey.CompletionsDebounce, undefined], + [ConfigKey.CompletionsDelay, undefined], + [ConfigKey.ModelAlwaysTerminatesSingleline, undefined], + [ConfigKey.UseWorkspaceContextCoordinator, undefined], + + + // These are only used for telemetry from LSP based editors and do not affect any behavior. + [ConfigKey.ShowEditorCompletions, undefined], + [ConfigKey.EnableAutoCompletions, undefined], + [ConfigKey.DelayCompletions, undefined], + [ConfigKey.FilterCompletions, undefined], + [ConfigKey.UseSplitContextPrompt, true], + + // These are defaults from package.json + [ConfigKey.Enable, { '*': true, 'plaintext': false, 'markdown': false, 'scminput': false }], + [ConfigKey.UserSelectedCompletionModel, ''], + + // These are advanced defaults from package.json + [ConfigKey.DebugOverrideEngineLegacy, ''], + [ConfigKey.DebugOverrideProxyUrlLegacy, ''], + [ConfigKey.DebugTestOverrideProxyUrlLegacy, ''], + [ConfigKey.DebugOverrideCapiUrlLegacy, ''], + [ConfigKey.DebugTestOverrideCapiUrlLegacy, ''], + [ConfigKey.DebugFilterLogCategories, []], +]); + +export function getConfig<T>(accessor: ServicesAccessor, key: ConfigKeyType): T { + return accessor.get(ICompletionsConfigProvider).getConfig(key); +} + +export function dumpForTelemetry(accessor: ServicesAccessor) { + try { + return accessor.get(ICompletionsConfigProvider).dumpForTelemetry(); + } catch (e) { + console.error(`Error dumping config for telemetry: ${e}`); + return {}; + } +} + +export class BuildInfo { + + static isPreRelease(): boolean { + return this.getBuildType() === BuildType.NIGHTLY; + } + + static isProduction(): boolean { + return this.getBuildType() !== BuildType.DEV; + } + + static getBuildType(): BuildType { + const buildType = <'dev' | 'prod'>packageJson.buildType; + if (buildType === 'prod') { + return BuildInfo.getVersion().length === 15 ? BuildType.NIGHTLY : BuildType.PROD; + } + return BuildType.DEV; + } + + static getVersion(): string { + return packageJson.version; + } + + static getBuild(): string { + return packageJson.build; + } +} + +type NameAndVersion = { + name: string; + version: string; +}; + +export type EditorInfo = NameAndVersion & { + // The root directory of the installation, currently only used to simplify stack traces. + root?: string; + // A programmatic name, used for error reporting. + devName?: string; +}; + +export type EditorPluginInfo = NameAndVersion; + +export type EditorPluginFilter = { filter: Filter; value: string; isVersion?: boolean }; + +export function formatNameAndVersion({ name, version }: NameAndVersion): string { + return `${name}/${version}`; +} + +export const ICompletionsEditorAndPluginInfo = createServiceIdentifier<ICompletionsEditorAndPluginInfo>('ICompletionsEditorAndPluginInfo'); +export interface ICompletionsEditorAndPluginInfo { + readonly _serviceBrand: undefined; + + getEditorInfo(): EditorInfo; + getEditorPluginInfo(): EditorPluginInfo; + getRelatedPluginInfo(): EditorPluginInfo[]; +} + +/** + * Do not use this in new code. Every endpoint has its own unique versioning. + * Centralizing in a single constant was a mistake. + * @deprecated + */ +export const apiVersion = '2025-05-01'; + +export function editorVersionHeaders(accessor: ServicesAccessor): { [key: string]: string } { + const info = accessor.get(ICompletionsEditorAndPluginInfo); + return { + 'Editor-Version': formatNameAndVersion(info.getEditorInfo()), + 'Editor-Plugin-Version': formatNameAndVersion(info.getEditorPluginInfo()), + 'Copilot-Language-Server-Version': BuildInfo.getVersion(), + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/constants.ts b/completions-sample-code/vscode-node/lib/src/constants.ts new file mode 100644 index 0000000..c13304e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/constants.ts @@ -0,0 +1,5 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export const CopilotConfigPrefix = 'github.copilot'; diff --git a/completions-sample-code/vscode-node/lib/src/defaultHandlers.ts b/completions-sample-code/vscode-node/lib/src/defaultHandlers.ts new file mode 100644 index 0000000..257a32c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/defaultHandlers.ts @@ -0,0 +1,41 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { Logger, logger } from './logger'; +import { isAbortError } from './networking'; +import { ICompletionsStatusReporter } from './progress'; + +const oomCodes = new Set(['ERR_WORKER_OUT_OF_MEMORY', 'ENOMEM']); + +function isOomError(error: NodeJS.ErrnoException) { + return ( + oomCodes.has(error.code ?? '') || + // happens in loadWasmLanguage + (error.name === 'RangeError' && error.message === 'WebAssembly.Memory(): could not allocate memory') + ); +} + +export function handleException(accessor: ServicesAccessor, err: unknown, origin: string, _logger: Logger = logger): void { + if (isAbortError(err)) { + // ignore cancelled fetch requests + return; + } + const statusReporter = accessor.get(ICompletionsStatusReporter); + if (err instanceof Error) { + const error = err as NodeJS.ErrnoException; + if (isOomError(error)) { + statusReporter.setWarning('Out of memory'); + } else if (error.code === 'EMFILE' || error.code === 'ENFILE') { + statusReporter.setWarning('Too many open files'); + } else if (error.code === 'CopilotPromptLoadFailure') { + statusReporter.setWarning('Corrupted Copilot installation'); + } else if (`${error.code}`.startsWith('CopilotPromptWorkerExit')) { + statusReporter.setWarning('Worker unexpectedly exited'); + } else if (error.syscall === 'uv_cwd' && error.code === 'ENOENT') { + statusReporter.setWarning('Current working directory does not exist'); + } + } + _logger.exception(accessor, err, origin); +} diff --git a/completions-sample-code/vscode-node/lib/src/diagnostics.ts b/completions-sample-code/vscode-node/lib/src/diagnostics.ts new file mode 100644 index 0000000..0b43014 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/diagnostics.ts @@ -0,0 +1,79 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { BuildInfo, ICompletionsEditorAndPluginInfo } from './config'; +import { TelemetryData } from './telemetry'; + +const os = { + EOL: '\n', +}; + +/** Diagnostics report made available in extension or agent. */ +interface Report { + sections: Section[]; +} + +type SectionItems = { [key: string]: boolean | string | number | undefined }; + +/** Section of a diagnostics report. */ +interface Section { + name: string; + items: SectionItems; +} + +export function collectCompletionDiagnostics(accessor: ServicesAccessor, telemetry: TelemetryData | undefined): Report { + const telemetryItems: SectionItems = {}; + if (telemetry !== undefined) { + if (telemetry.properties.headerRequestId) { + telemetryItems['Header Request ID'] = telemetry.properties.headerRequestId; + } + if (telemetry.properties.choiceIndex) { + telemetryItems['Choice Index'] = telemetry.properties.choiceIndex; + } + if (telemetry.properties.opportunityId) { + telemetryItems['Opportunity ID'] = telemetry.properties.opportunityId; + } + if (telemetry.properties.clientCompletionId) { + telemetryItems['Client Completion ID'] = telemetry.properties.clientCompletionId; + } + if (telemetry.properties.engineName) { + telemetryItems['Model ID'] = telemetry.properties.engineName; + } + } + return { + sections: [ + { + name: 'Copilot Extension', + items: { + Version: BuildInfo.getVersion(), + Editor: getEditorDisplayVersion(accessor), + ...telemetryItems, + }, + }, + ], + }; +} + +export function formatDiagnosticsAsMarkdown(data: Report): string { + const s = data.sections.map(formatSectionAsMarkdown); + return s.join(os.EOL + os.EOL) + os.EOL; +} + +function formatSectionAsMarkdown(s: Section) { + return ( + `## ${s.name}` + + os.EOL + + os.EOL + + Object.keys(s.items) + .filter(k => k !== 'name') + .map(k => `- ${k}: ${s.items[k] ?? 'N/A'}`) + .join(os.EOL) + ); +} + +function getEditorDisplayVersion(accessor: ServicesAccessor): string { + const info = accessor.get(ICompletionsEditorAndPluginInfo).getEditorInfo(); + return `${info.name} ${info.version}`; +} diff --git a/completions-sample-code/vscode-node/lib/src/documentTracker.ts b/completions-sample-code/vscode-node/lib/src/documentTracker.ts new file mode 100644 index 0000000..96b3ebc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/documentTracker.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { LRUCacheMap } from './helpers/cache'; +import { TextDocumentIdentifier } from './textDocument'; +import { ICompletionsTextDocumentManagerService } from './textDocumentManager'; + +/** + * A map from the string representation of a document URI to its last access time in ms since the + * epoch. + */ +export const accessTimes = new LRUCacheMap<string, number>(); + +/** + * Returns a copy of `docs` sorted by access time, from most to least recent. + */ +export function sortByAccessTimes<T extends TextDocumentIdentifier>(docs: readonly T[]): T[] { + return [...docs].sort((a, b) => { + const aAccessTime = accessTimes.get(a.uri) ?? 0; + const bAccessTime = accessTimes.get(b.uri) ?? 0; + return bAccessTime - aAccessTime; + }); +} + +/** + * Registers a listener on the `window.onDidChangeActiveTextEditor` event that records/updates the + * access time of the document. + */ +export const registerDocumentTracker = (accessor: ServicesAccessor) => + accessor.get(ICompletionsTextDocumentManagerService).onDidFocusTextDocument(e => { + if (e.document) { + accessTimes.set(e.document.uri.toString(), Date.now()); + } + }); diff --git a/completions-sample-code/vscode-node/lib/src/error/userErrorNotifier.ts b/completions-sample-code/vscode-node/lib/src/error/userErrorNotifier.ts new file mode 100644 index 0000000..bd1cd89 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/error/userErrorNotifier.ts @@ -0,0 +1,61 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IEnvService } from '../../../../../../platform/env/common/envService'; +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { URI } from '../../../../../../util/vs/base/common/uri'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { ICompletionsNotificationSender } from '../notificationSender'; + +const CERTIFICATE_ERRORS = ['UNABLE_TO_VERIFY_LEAF_SIGNATURE', 'CERT_SIGNATURE_FAILURE']; +const errorMsg = + 'Your proxy connection requires a trusted certificate. Please make sure the proxy certificate and any issuers are configured correctly and trusted by your operating system.'; +const learnMoreLink = 'https://gh.io/copilot-network-errors'; + +export const ICompletionsUserErrorNotifierService = createServiceIdentifier<ICompletionsUserErrorNotifierService>('ICompletionsUserErrorNotifierService'); +export interface ICompletionsUserErrorNotifierService { + readonly _serviceBrand: undefined; + notifyUser(e: unknown): void; +} + +export class UserErrorNotifier implements ICompletionsUserErrorNotifierService { + declare _serviceBrand: undefined; + private readonly notifiedErrorCodes: string[] = []; + + constructor( + @ICompletionsLogTargetService private readonly _logTarget: ICompletionsLogTargetService, + @ICompletionsNotificationSender private readonly _notificationSender: ICompletionsNotificationSender, + @IEnvService private readonly _env: IEnvService + ) { } + + notifyUser(e: unknown) { + if (!(e instanceof Error)) { return; } + const error: NodeJS.ErrnoException = e; + if (error.code && CERTIFICATE_ERRORS.includes(error.code) && !this.didNotifyBefore(error.code)) { + this.notifiedErrorCodes.push(error.code); + void this.displayCertificateErrorNotification(error); + } + } + + private async displayCertificateErrorNotification(err: NodeJS.ErrnoException) { + new Logger('certificates').error( + this._logTarget, + `${errorMsg} Please visit ${learnMoreLink} to learn more. Original cause:`, + err + ); + const learnMoreAction = { title: 'Learn more' }; + return this._notificationSender + .showWarningMessage(errorMsg, learnMoreAction) + .then(userResponse => { + if (userResponse?.title === learnMoreAction.title) { + return this._env.openExternal(URI.parse(learnMoreLink)); + } + }); + } + + private didNotifyBefore(code: string) { + return this.notifiedErrorCodes.indexOf(code) !== -1; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/defaultExpFilters.ts b/completions-sample-code/vscode-node/lib/src/experiments/defaultExpFilters.ts new file mode 100644 index 0000000..1824c7f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/defaultExpFilters.ts @@ -0,0 +1,76 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { IExperimentationService } from '../../../../../../platform/telemetry/common/nullExperimentationService'; +import { IDisposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotToken } from '../auth/copilotTokenManager'; +import { getUserKind } from '../auth/orgs'; +import { + BuildInfo, + BuildType, + ConfigKey, + getConfig +} from '../config'; +import { getEngineRequestInfo } from '../openai/config'; +import { Filter, Release } from './filters'; + +export function setupCompletionsExperimentationService(accessor: ServicesAccessor): IDisposable { + const authService = accessor.get(IAuthenticationService); + const instantiationService = accessor.get(IInstantiationService); + + const disposable = authService.onDidAccessTokenChange(() => { + authService.getCopilotToken() + .then(t => instantiationService.invokeFunction(updateCompletionsFilters, t)) + .catch(err => { }); + }); + + updateCompletionsFilters(accessor, authService.copilotToken); + + return disposable; +} + +function getPluginRelease(accessor: ServicesAccessor): Release { + if (BuildInfo.getBuildType() === BuildType.NIGHTLY) { + return Release.Nightly; + } + return Release.Stable; +} + +function updateCompletionsFilters(accessor: ServicesAccessor, token: Omit<CopilotToken, 'token'> | undefined) { + const exp = accessor.get(IExperimentationService); + + const filters = createCompletionsFilters(accessor, token); + + exp.setCompletionsFilters(filters); +} + +export function createCompletionsFilters(accessor: ServicesAccessor, token: Omit<CopilotToken, 'token'> | undefined) { + const filters = new Map<Filter, string>(); + + filters.set(Filter.ExtensionRelease, getPluginRelease(accessor)); + filters.set(Filter.CopilotOverrideEngine, getConfig(accessor, ConfigKey.DebugOverrideEngine) || getConfig(accessor, ConfigKey.DebugOverrideEngineLegacy)); + filters.set(Filter.CopilotClientVersion, BuildInfo.isProduction() ? BuildInfo.getVersion() : '1.999.0'); + + if (token) { + const userKind = getUserKind(token); + const customModel = token.getTokenValue('ft') ?? ''; + const orgs = token.getTokenValue('ol') ?? ''; + const customModelNames = token.getTokenValue('cml') ?? ''; + const copilotTrackingId = token.getTokenValue('tid') ?? ''; + + filters.set(Filter.CopilotUserKind, userKind); + filters.set(Filter.CopilotCustomModel, customModel); + filters.set(Filter.CopilotOrgs, orgs); + filters.set(Filter.CopilotCustomModelNames, customModelNames); + filters.set(Filter.CopilotTrackingId, copilotTrackingId); + filters.set(Filter.CopilotUserKind, getUserKind(token)); + } + + const model = getEngineRequestInfo(accessor).modelId; + filters.set(Filter.CopilotEngine, model); + return filters; +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/experiments/expConfig.ts b/completions-sample-code/vscode-node/lib/src/experiments/expConfig.ts new file mode 100644 index 0000000..f8e58b1 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/expConfig.ts @@ -0,0 +1,154 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TelemetryData, telemetryExpProblem } from '../telemetry'; +import { ExpServiceTelemetryNames } from './telemetryNames'; + +// All variables we pull from Exp and might want to use +export enum ExpTreatmentVariables { + // the engine we want to request, used in actual experiment(s) + CustomEngine = 'copilotcustomengine', + // if set, any custom engine (see previous) will only apply when the current engine matches the value of this variable + CustomEngineTargetEngine = 'copilotcustomenginetargetengine', + + OverrideBlockMode = 'copilotoverrideblockmode', + SuffixPercent = 'CopilotSuffixPercent', // the percentage of the prompt tokens to allocate to the suffix + CppHeadersEnableSwitch = 'copilotcppheadersenableswitch', // whether to enable the inclusion of C++ headers as neighbors in the prompt + UseSubsetMatching = 'copilotsubsetmatching', // whether to use subset matching instead of jaccard similarity experiment + + // granularity specification + SuffixMatchThreshold = 'copilotsuffixmatchthreshold', // the threshold that new suffix should match with old suffix + + MaxPromptCompletionTokens = 'maxpromptcompletionTokens', // the maximum tokens of the prompt and completion + + /** + * Enable the use of the Workspace Context Coordinator to coordinate context from providers of workspace snippets. + */ + StableContextPercent = 'copilotstablecontextpercent', // the percentage of the prompt tokens to allocate to the stable context + VolatileContextPercent = 'copilotvolatilecontextpercent', // the percentage of the prompt tokens to allocate to the volatile context + + /** + * Flags that control the enablement of the related files extensibility for various languages in VSCode. + */ + RelatedFilesVSCodeCSharp = 'copilotrelatedfilesvscodecsharp', // whether to include related files as neighbors in the prompt for C#, this takes precedence over RelatedFilesVSCode + RelatedFilesVSCodeTypeScript = 'copilotrelatedfilesvscodetypescript', // whether to include related files as neighbors in the prompt for TS/JS, this takes precedence over RelatedFilesVSCode + RelatedFilesVSCode = 'copilotrelatedfilesvscode', // whether to include related files as neighbors in the prompt, vscode experiment + + /** + * Flags that control the inclusion of open tab files as neighboring files for various languages. + */ + ContextProviders = 'copilotcontextproviders', // comma-separated list of context providers IDs (case sensitive) to enable + IncludeNeighboringFiles = 'copilotincludeneighboringfiles', // Always include neighboring files alongside context providers + ExcludeRelatedFiles = 'copilotexcluderelatedfiles', // Exclude related files even if neighboring files are enabled + ContextProviderTimeBudget = 'copilotcontextprovidertimebudget', // time budget for context providers in milliseconds + + /** + * Values to control the ContextProvider API's CodeSnippets provided by the C++ Language Service. + */ + CppContextProviderParams = 'copilotcppContextProviderParams', + + /** + * Values to control the ContextProvider API's CodeSnippets provided by the C# Language Service. + */ + CSharpContextProviderParams = 'copilotcsharpcontextproviderparams', + + /** + * Values to control the ContextProvider API's CodeSnippets provided by the Java Language Service. + */ + JavaContextProviderParams = 'copilotjavacontextproviderparams', + + /** + * Values to control the MultiLanguageContextProvider parameters. + */ + MultiLanguageContextProviderParams = 'copilotmultilanguagecontextproviderparams', + + /** + * Values to control the TsContextProvider parameters. + */ + TsContextProviderParams = 'copilottscontextproviderparams', + + /** + * Controls the delay to apply to debouncing of completion requests. + */ + CompletionsDebounce = 'copilotcompletionsdebounce', + + /** + * Enable the electron networking in VS Code. + */ + ElectronFetcher = 'copilotelectronfetcher', + FetchFetcher = 'copilotfetchfetcher', + + /** + * Sets the timeout for waiting for async completions in flight before + * issuing a new network request. Set to -1 to disable the timeout entirely. + */ + AsyncCompletionsTimeout = 'copilotasynccompletionstimeout', + + /** + * Controls whether the prompt context for code completions needs to be split from the document prefix. + */ + EnablePromptContextProxyField = 'copilotenablepromptcontextproxyfield', + + /** + * Controls progressive reveal of completions. + */ + ProgressiveReveal = 'copilotprogressivereveal', + // part of progressive reveal, controls whether the model or client terminates single-line completions + ModelAlwaysTerminatesSingleline = 'copilotmodelterminatesingleline', + // long look-ahead window size (in lines) for progressive reveal + ProgressiveRevealLongLookaheadSize = 'copilotprogressivereveallonglookaheadsize', + // short look-ahead window size (in lines) for progressive reveal + ProgressiveRevealShortLookaheadSize = 'copilotprogressiverevealshortlookaheadsize', + // maximum token count when requesting multi-line completions + MaxMultilineTokens = 'copilotmaxmultilinetokens', + + /** + * Controls number of lines to trim to after accepting a completion. + */ + MultilineAfterAcceptLines = 'copilotmultilineafteracceptlines', + + /** + * Add a delay before rendering completions. + */ + CompletionsDelay = 'copilotcompletionsdelay', + + /** + * Request single line completions unless the previous completion was just accepted. + */ + SingleLineUnlessAccepted = 'copilotsinglelineunlessaccepted', +} + +export type ExpTreatmentVariableValue = boolean | string | number; + +export class ExpConfig { + variables: Partial<Record<ExpTreatmentVariables, ExpTreatmentVariableValue>>; // for the 'vscode' config + features: string; // semicolon-separated feature IDs + + constructor( + variables: Partial<Record<ExpTreatmentVariables, ExpTreatmentVariableValue>>, + features: string + ) { + this.variables = variables; + this.features = features; + } + + static createFallbackConfig(accessor: ServicesAccessor, reason: string): ExpConfig { + telemetryExpProblem(accessor, { reason }); + return this.createEmptyConfig(); + } + + static createEmptyConfig() { + return new ExpConfig({}, ''); + } + + /** + * Adds (or overwrites) the given experiment config to the telemetry data. + * @param telemetryData telemetryData object. If previous ExpConfigs are already present, they will be overwritten. + */ + addToTelemetry(telemetryData: TelemetryData): void { + telemetryData.properties[ExpServiceTelemetryNames.featuresTelemetryPropertyName] = this.features; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/features.ts b/completions-sample-code/vscode-node/lib/src/experiments/features.ts new file mode 100644 index 0000000..de285d9 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/features.ts @@ -0,0 +1,415 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ILogService } from '../../../../../../platform/log/common/logService'; +import { IExperimentationService } from '../../../../../../platform/telemetry/common/nullExperimentationService'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + DEFAULT_MAX_COMPLETION_LENGTH, + DEFAULT_MAX_PROMPT_LENGTH, + DEFAULT_PROMPT_ALLOCATION_PERCENT, + DEFAULT_SUFFIX_MATCH_THRESHOLD +} from '../../../prompt/src/prompt'; +import { CopilotToken, ICompletionsCopilotTokenManager } from '../auth/copilotTokenManager'; +import { BlockMode } from '../config'; +import { TelemetryData, TelemetryWithExp } from '../telemetry'; +import { createCompletionsFilters } from './defaultExpFilters'; +import { ExpConfig, ExpTreatmentVariables, ExpTreatmentVariableValue } from './expConfig'; +import { CompletionsFiltersInfo, ContextProviderExpSettings, ICompletionsFeaturesService } from './featuresService'; +import { Filter, FilterSettings } from './filters'; + +type InternalContextProviderExpSettings = { + id?: string; + ids?: string[]; + includeNeighboringFiles?: boolean; + excludeRelatedFiles?: boolean; + timeBudget?: number; + params?: Record<string, string | boolean | number>; +}; + +/** General-purpose API for accessing ExP variable values. */ +export class Features implements ICompletionsFeaturesService { + declare _serviceBrand: undefined; + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @IExperimentationService private readonly experimentationService: IExperimentationService, + @ICompletionsCopilotTokenManager private readonly copilotTokenManager: ICompletionsCopilotTokenManager, + ) { } + + /** + * Central logic for obtaining the assignments of treatment groups + * for a given set of filters (i.e. descriptors of who is getting the treatment). + * Also gets the values of variables controlled by experiment. + * + * This function should be called **exactly once** at the start of every + * 'completion request' in the client (e.g. ghostText, panel request or chat conversation). + * + * It is called with an initial set of filters, (FeaturesFilterArgs) + * but it adds many of its own. + * At first the general background filters like extension version. + * Then it will check ExP assignments for the first time, to find out + * whether there are any assignments of a special granularity + * (i.e. the concept that we want to redraw assignments based on + * time bucket, or checksum of time, etc). + * + * On most calls to this function, the assignment fetches will be the + * assignments from previously used filters, so they will be cached and return fast. + * + * @param telemetryData The base telemetry object to which the experimental filters, ExP + * variable values, and experimental assignments will be added. All properties and measurements + * of the input telemetryData will be present in the output TelemetryWithExp object. + * Every telemetry data used to generate ExP scorecards (e.g. ghostText events) must + * include the correct experiment assignments in order to properly create those + * scorecards. + */ + async updateExPValuesAndAssignments( + filtersInfo?: CompletionsFiltersInfo, + telemetryData: TelemetryData = TelemetryData.createAndMarkAsIssued() + ): Promise<TelemetryWithExp> { + // We should not allow accidentally overwriting existing ExP vals/assignments. + // This doesn't stop all misuse cases, but should prevent some trivial ones. + if (telemetryData instanceof TelemetryWithExp) { + throw new Error('updateExPValuesAndAssignments should not be called with TelemetryWithExp'); + } + + const token = this.copilotTokenManager.token ?? await this.copilotTokenManager.getToken(); + const { filters, exp } = this.createExpConfigAndFilters(token); + + return new TelemetryWithExp(telemetryData.properties, telemetryData.measurements, telemetryData.issuedTime, { + filters, + exp: exp, + }); + } + + /** + * Request a Copilot token and use that token to call updateExPValuesAndAssignments. Do NOT call this at startup. + * Instead, register a onCopilotToken handler and use that token with updateExPValuesAndAssignments directly. + */ + async fetchTokenAndUpdateExPValuesAndAssignments( + filtersInfo?: CompletionsFiltersInfo, + telemetryData?: TelemetryData + ) { + return await this.updateExPValuesAndAssignments(filtersInfo, telemetryData); + } + + private createExpConfigAndFilters(token: CopilotToken) { + + const exp2: Partial<Record<ExpTreatmentVariables, ExpTreatmentVariableValue>> = {}; + for (const varName of Object.values<ExpTreatmentVariables>(ExpTreatmentVariables)) { + const value = this.experimentationService.getTreatmentVariable(varName); + if (value !== undefined) { + exp2[varName] = value; + } + } + + const features = Object.entries(exp2).map(([name, value]) => { + // Based on what tas-client does in https://github.com/microsoft/tas-client/blob/2bd24c976273b671892aad99139af2c7c7dc3b26/tas-client/src/tas-client/FeatureProvider/TasApiFeatureProvider.ts#L59 + return name + (value ? '' : 'cf'); + }); + const exp = new ExpConfig(exp2, features.join(';')); + const filterMap = this.instantiationService.invokeFunction(createCompletionsFilters, token); + const filterRecord: Partial<Record<Filter, string>> = {}; + for (const [key, value] of filterMap.entries()) { + filterRecord[key] = value; + } + + const filters = new FilterSettings(filterRecord); + return { filters, exp }; + } + + /** Get the entries from this.assignments corresponding to given settings. */ + async getFallbackExpAndFilters(): Promise<{ filters: FilterSettings; exp: ExpConfig }> { + const token = this.copilotTokenManager.token ?? await this.copilotTokenManager.getToken(); + return this.createExpConfigAndFilters(token); + } + + /** Override for BlockMode to send in the request. */ + overrideBlockMode(telemetryWithExp: TelemetryWithExp): BlockMode | undefined { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.OverrideBlockMode] as BlockMode) || + undefined + ); + } + + /** Functions with arguments, passed via object destructuring */ + + /** @returns the string for copilotcustomengine, or "" if none is set. */ + customEngine(telemetryWithExp: TelemetryWithExp): string { + return (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.CustomEngine] as string) ?? ''; + } + + /** @returns the string for copilotcustomenginetargetengine, or undefined if none is set. */ + customEngineTargetEngine(telemetryWithExp: TelemetryWithExp): string | undefined { + return telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.CustomEngineTargetEngine] as string; + } + + /** @returns the percent of prompt tokens to be allocated to the suffix */ + suffixPercent(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.SuffixPercent] as number) ?? + DEFAULT_PROMPT_ALLOCATION_PERCENT.suffix + ); + } + + /** @returns the percentage match threshold for using the cached suffix */ + suffixMatchThreshold(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.SuffixMatchThreshold] as number) ?? + DEFAULT_SUFFIX_MATCH_THRESHOLD + ); + } + + /** @returns whether to enable the inclusion of C++ headers as neighbor files. */ + cppHeadersEnableSwitch(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.CppHeadersEnableSwitch] as boolean) ?? + false + ); + } + + /** @returns whether to use included related files as neighbor files for C# (vscode experiment). */ + relatedFilesVSCodeCSharp(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] as boolean) ?? + false + ); + } + + /** @returns whether to use included related files as neighbor files for TS/JS (vscode experiment). */ + relatedFilesVSCodeTypeScript(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.RelatedFilesVSCodeTypeScript + ] as boolean) ?? false + ); + } + + /** @returns whether to use included related files as neighbor files (vscode experiment). */ + relatedFilesVSCode(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCode] as boolean) ?? false + ); + } + + /** @returns the list of context providers IDs to enable. The special value `*` enables all context providers. */ + contextProviders(telemetryWithExp: TelemetryWithExp): string[] { + const providers = (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.ContextProviders] ?? + '') as string; + if (!providers) { + return []; + } + return providers.split(',').map(provider => provider.trim()); + } + + contextProviderTimeBudget(languageId: string, telemetryWithExp: TelemetryWithExp): number { + const client = ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.ContextProviderTimeBudget] as number) ?? + 150 + ); + if (client) { + return client; + } + const chat = this.getContextProviderExpSettings(languageId); + return chat?.timeBudget ?? 150; + } + + includeNeighboringFiles(languageId: string, telemetryWithExp: TelemetryWithExp): boolean { + const client = ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.IncludeNeighboringFiles] as boolean) ?? + false + ); + if (client) { + return true; + } + const chat = this.getContextProviderExpSettings(languageId); + return chat?.includeNeighboringFiles ?? false; + } + + excludeRelatedFiles(languageId: string, telemetryWithExp: TelemetryWithExp): boolean { + const client = ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.ExcludeRelatedFiles] as boolean) ?? + false + ); + if (client) { + return true; + } + const chat = this.getContextProviderExpSettings(languageId); + return chat?.excludeRelatedFiles ?? false; + } + + getContextProviderExpSettings(languageId: string): ContextProviderExpSettings | undefined { + const value = this.experimentationService.getTreatmentVariable<string>(`config.github.copilot.chat.contextprovider.${languageId}`); + if (typeof value === 'string') { + try { + const parsed: Partial<InternalContextProviderExpSettings> = JSON.parse(value); + const ids = this.getProviderIDs(parsed); + delete parsed.id; + delete parsed.ids; + return Object.assign({ ids }, { includeNeighboringFiles: false, excludeRelatedFiles: false, timeBudget: 150 }, parsed as Omit<InternalContextProviderExpSettings, 'id' | 'ids'>); + } catch (err) { + this.instantiationService.invokeFunction((accessor) => { + const logService = accessor.get(ILogService); + logService.error(`Failed to parse context provider exp settings for language ${languageId}`); + }); + return undefined; + } + } else { + return undefined; + } + } + + private getProviderIDs(json: InternalContextProviderExpSettings): string[] { + const result: string[] = []; + if (typeof json.id === 'string' && json.id.length > 0) { + result.push(json.id); + } + if (Array.isArray(json.ids)) { + for (const id of json.ids) { + if (typeof id === 'string' && id.length > 0) { + result.push(id); + } + } + } + return result; + } + + /** @returns the maximal number of tokens of prompt AND completion */ + maxPromptCompletionTokens(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.MaxPromptCompletionTokens] as number) ?? + DEFAULT_MAX_PROMPT_LENGTH + DEFAULT_MAX_COMPLETION_LENGTH + ); + } + + stableContextPercent(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.StableContextPercent] as number) ?? + DEFAULT_PROMPT_ALLOCATION_PERCENT.stableContext + ); + } + + volatileContextPercent(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.VolatileContextPercent] as number) ?? + DEFAULT_PROMPT_ALLOCATION_PERCENT.volatileContext + ); + } + + /** Custom parameters for language specific Context Providers. */ + cppContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined { + const cppContextProviderParams = telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.CppContextProviderParams + ] as string; + return cppContextProviderParams; + } + + csharpContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined { + const csharpContextProviderParams = telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.CSharpContextProviderParams + ] as string; + return csharpContextProviderParams; + } + + javaContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined { + const javaContextProviderParams = telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.JavaContextProviderParams + ] as string; + return javaContextProviderParams; + } + + multiLanguageContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined { + const multiLanguageContextProviderParams = telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.MultiLanguageContextProviderParams + ] as string; + return multiLanguageContextProviderParams; + } + + tsContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined { + const tsContextProviderParams = telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.TsContextProviderParams + ] as string; + return tsContextProviderParams; + } + + completionsDebounce(telemetryWithExp: TelemetryWithExp): number | undefined { + return telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.CompletionsDebounce] as + | number + | undefined; + } + + enableElectronFetcher(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.ElectronFetcher] as boolean) ?? false + ); + } + + enableFetchFetcher(telemetryWithExp: TelemetryWithExp): boolean { + return (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.FetchFetcher] as boolean) ?? false; + } + + asyncCompletionsTimeout(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.AsyncCompletionsTimeout] as number) ?? + 200 + ); + } + + enableProgressiveReveal(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.ProgressiveReveal] as boolean) ?? false + ); + } + + modelAlwaysTerminatesSingleline(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.ModelAlwaysTerminatesSingleline + ] as boolean) ?? true + ); + } + + longLookaheadSize(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.ProgressiveRevealLongLookaheadSize + ] as number) ?? 9 + ); + } + + shortLookaheadSize(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ + ExpTreatmentVariables.ProgressiveRevealShortLookaheadSize + ] as number) ?? 3 + ); + } + + maxMultilineTokens(telemetryWithExp: TelemetryWithExp): number { + // p50 line length is 19 characters (p95 is 73) + // average token length is around 4 characters + // the below value has quite a bit of buffer while bringing the limit in significantly from 500 + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.MaxMultilineTokens] as number) ?? 200 + ); + } + + multilineAfterAcceptLines(telemetryWithExp: TelemetryWithExp): number { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.MultilineAfterAcceptLines] as number) ?? + 1 + ); + } + + completionsDelay(telemetryWithExp: TelemetryWithExp): number { + return (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.CompletionsDelay] as number) ?? 200; + } + + singleLineUnlessAccepted(telemetryWithExp: TelemetryWithExp): boolean { + return ( + (telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.SingleLineUnlessAccepted] as boolean) ?? + false + ); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/featuresService.ts b/completions-sample-code/vscode-node/lib/src/experiments/featuresService.ts new file mode 100644 index 0000000..89bf712 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/featuresService.ts @@ -0,0 +1,68 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { BlockMode } from '../config'; +import { TelemetryData, TelemetryWithExp } from '../telemetry'; +import { ExpConfig } from './expConfig'; +import { FilterSettings } from './filters'; + +export type CompletionsFiltersInfo = { uri: string; languageId: string }; + +export type ContextProviderExpSettings = { + ids: string[]; + includeNeighboringFiles: boolean; + excludeRelatedFiles: boolean; + timeBudget: number; + params?: Record<string, string | boolean | number>; +} + +export const ICompletionsFeaturesService = createServiceIdentifier<ICompletionsFeaturesService>('ICompletionsFeaturesService'); +export interface ICompletionsFeaturesService { + readonly _serviceBrand: undefined; + updateExPValuesAndAssignments( + filtersInfo?: CompletionsFiltersInfo, + telemetryData?: TelemetryData + ): Promise<TelemetryWithExp>; + fetchTokenAndUpdateExPValuesAndAssignments( + filtersInfo?: CompletionsFiltersInfo, + telemetryData?: TelemetryData + ): Promise<TelemetryWithExp>; + getFallbackExpAndFilters(): Promise<{ filters: FilterSettings; exp: ExpConfig }>; + overrideBlockMode(telemetryWithExp: TelemetryWithExp): BlockMode | undefined; + customEngine(telemetryWithExp: TelemetryWithExp): string; + customEngineTargetEngine(telemetryWithExp: TelemetryWithExp): string | undefined; + suffixPercent(telemetryWithExp: TelemetryWithExp): number; + suffixMatchThreshold(telemetryWithExp: TelemetryWithExp): number; + cppHeadersEnableSwitch(telemetryWithExp: TelemetryWithExp): boolean; + relatedFilesVSCodeCSharp(telemetryWithExp: TelemetryWithExp): boolean; + relatedFilesVSCodeTypeScript(telemetryWithExp: TelemetryWithExp): boolean; + relatedFilesVSCode(telemetryWithExp: TelemetryWithExp): boolean; + contextProviders(telemetryWithExp: TelemetryWithExp): string[]; + contextProviderTimeBudget(languageId: string, telemetryWithExp: TelemetryWithExp): number; + includeNeighboringFiles(languageId: string, telemetryWithExp: TelemetryWithExp): boolean; + excludeRelatedFiles(languageId: string, telemetryWithExp: TelemetryWithExp): boolean; + getContextProviderExpSettings(languageId: string): ContextProviderExpSettings | undefined; + maxPromptCompletionTokens(telemetryWithExp: TelemetryWithExp): number; + stableContextPercent(telemetryWithExp: TelemetryWithExp): number; + volatileContextPercent(telemetryWithExp: TelemetryWithExp): number; + cppContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined; + csharpContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined; + javaContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined; + multiLanguageContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined; + tsContextProviderParams(telemetryWithExp: TelemetryWithExp): string | undefined; + completionsDebounce(telemetryWithExp: TelemetryWithExp): number | undefined; + enableElectronFetcher(telemetryWithExp: TelemetryWithExp): boolean; + enableFetchFetcher(telemetryWithExp: TelemetryWithExp): boolean; + asyncCompletionsTimeout(telemetryWithExp: TelemetryWithExp): number; + enableProgressiveReveal(telemetryWithExp: TelemetryWithExp): boolean; + modelAlwaysTerminatesSingleline(telemetryWithExp: TelemetryWithExp): boolean; + longLookaheadSize(telemetryWithExp: TelemetryWithExp): number; + shortLookaheadSize(telemetryWithExp: TelemetryWithExp): number; + maxMultilineTokens(telemetryWithExp: TelemetryWithExp): number; + multilineAfterAcceptLines(telemetryWithExp: TelemetryWithExp): number; + completionsDelay(telemetryWithExp: TelemetryWithExp): number; + singleLineUnlessAccepted(telemetryWithExp: TelemetryWithExp): boolean; +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/experiments/filters.ts b/completions-sample-code/vscode-node/lib/src/experiments/filters.ts new file mode 100644 index 0000000..331b112 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/filters.ts @@ -0,0 +1,105 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TelemetryData } from '../telemetry'; + +/** The prefix used for related plugin version headers. */ +const CopilotRelatedPluginVersionPrefix = 'X-Copilot-RelatedPluginVersion-'; + +/** The filter headers that ExP knows about. */ +export enum Filter { + // Default VSCode filters + + ExtensionRelease = 'X-VSCode-ExtensionRelease', + + // Copilot-specific filters + + /** The machine ID concatenated with a 1-hour bucket. */ + CopilotClientTimeBucket = 'X-Copilot-ClientTimeBucket', + /** The model currently in use. Not included in fallback filters */ + CopilotEngine = 'X-Copilot-Engine', + /** The engine override value from settings, if present. */ + CopilotOverrideEngine = 'X-Copilot-OverrideEngine', + /** Git repo info. Not included in fallback filters */ + CopilotRepository = 'X-Copilot-Repository', + /** Language of the file on which a given request is being made. Not included in fallback filters */ + CopilotFileType = 'X-Copilot-FileType', // Wired to languageId + /** The organization the user belongs to. Not included in fallback filters */ + CopilotUserKind = 'X-Copilot-UserKind', + /** Declare experiment dogfood program if any. Not included in fallback filters */ + CopilotDogfood = 'X-Copilot-Dogfood', + /** For custom Model Alpha. Not included in fallback filters */ + CopilotCustomModel = 'X-Copilot-CustomModel', + /** Organizations. */ + CopilotOrgs = 'X-Copilot-Orgs', + /** Identifiers for Custom Model(s) */ + CopilotCustomModelNames = 'X-Copilot-CustomModelNames', + /** Copilot Tracking ID */ + CopilotTrackingId = 'X-Copilot-CopilotTrackingId', + /** The Copilot Client Version */ + CopilotClientVersion = 'X-Copilot-ClientVersion', + + CopilotRelatedPluginVersionCppTools = CopilotRelatedPluginVersionPrefix + 'msvscodecpptools', + CopilotRelatedPluginVersionCMakeTools = CopilotRelatedPluginVersionPrefix + 'msvscodecmaketools', + CopilotRelatedPluginVersionMakefileTools = CopilotRelatedPluginVersionPrefix + 'msvscodemakefiletools', + CopilotRelatedPluginVersionCSharpDevKit = CopilotRelatedPluginVersionPrefix + 'msdotnettoolscsdevkit', + CopilotRelatedPluginVersionPython = CopilotRelatedPluginVersionPrefix + 'mspythonpython', + CopilotRelatedPluginVersionPylance = CopilotRelatedPluginVersionPrefix + 'mspythonvscodepylance', + CopilotRelatedPluginVersionJavaPack = CopilotRelatedPluginVersionPrefix + 'vscjavavscodejavapack', + CopilotRelatedPluginVersionJavaManager = CopilotRelatedPluginVersionPrefix + 'vscjavavscodejavadependency', + CopilotRelatedPluginVersionTypescript = CopilotRelatedPluginVersionPrefix + 'vscodetypescriptlanguagefeatures', + CopilotRelatedPluginVersionTypescriptNext = CopilotRelatedPluginVersionPrefix + 'msvscodevscodetypescriptnext', + CopilotRelatedPluginVersionCSharp = CopilotRelatedPluginVersionPrefix + 'msdotnettoolscsharp', + CopilotRelatedPluginVersionGithubCopilotChat = CopilotRelatedPluginVersionPrefix + 'githubcopilotchat', + CopilotRelatedPluginVersionGithubCopilot = CopilotRelatedPluginVersionPrefix + 'githubcopilot', +} + +export enum Release { + Stable = 'stable', + Nightly = 'nightly', +} + +const telmetryNames: Partial<Record<Filter, string>> = { + [Filter.CopilotClientTimeBucket]: 'timeBucket', + [Filter.CopilotOverrideEngine]: 'engine', + [Filter.CopilotRepository]: 'repo', + [Filter.CopilotFileType]: 'fileType', + [Filter.CopilotUserKind]: 'userKind', +}; + +/** + * The class FilterSettings holds the variables that were used to filter + * experiment groups. + */ +export class FilterSettings { + constructor(private readonly filters: Partial<Record<Filter, string>>) { + // empyt string is equivalent to absent, so remove it + for (const [filter, value] of Object.entries(this.filters)) { + if (value === '') { + delete this.filters[filter as Filter]; + } + } + } + + /** + * Extends the telemetry Data with the current filter variables. + * @param telemetryData Extended in place. + */ + addToTelemetry(telemetryData: TelemetryData) { + // add all values: + for (const [filter, value] of Object.entries(this.filters)) { + const telemetryName = telmetryNames[filter as Filter]; + if (telemetryName === undefined) { + continue; + } + telemetryData.properties[telemetryName] = value; + } + } + + /** Returns a copy of the filters. */ + toHeaders(): Partial<Record<Filter, string>> { + return { ...this.filters }; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/similarFileOptionsProvider.ts b/completions-sample-code/vscode-node/lib/src/experiments/similarFileOptionsProvider.ts new file mode 100644 index 0000000..c97610b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/similarFileOptionsProvider.ts @@ -0,0 +1,50 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { DEFAULT_NUM_SNIPPETS } from '../../../prompt/src/prompt'; +import { defaultSimilarFilesOptions, SimilarFilesOptions } from '../../../prompt/src/snippetInclusion/similarFiles'; +import { ConfigKey, getConfig } from '../config'; +import { TelemetryWithExp } from '../telemetry'; +import { ExpTreatmentVariables } from './expConfig'; +import { getCppNumberOfSnippets, getCppSimilarFilesOptions } from './similarFileOptionsProviderCpp'; + +type SimilarFilesOptionsProvider = (accessor: ServicesAccessor, exp: TelemetryWithExp) => SimilarFilesOptions; +// Add here for more options for other language ids. +const languageSimilarFilesOptions: ReadonlyMap<string, SimilarFilesOptionsProvider> = new Map< + string, + SimilarFilesOptionsProvider +>([['cpp', getCppSimilarFilesOptions]]); + +export function getSimilarFilesOptions(accessor: ServicesAccessor, exp: TelemetryWithExp, langId: string): SimilarFilesOptions { + const optionsProvider: SimilarFilesOptionsProvider | undefined = languageSimilarFilesOptions.get(langId); + if (optionsProvider) { + return optionsProvider(accessor, exp); + } else { + return { + ...defaultSimilarFilesOptions, + useSubsetMatching: useSubsetMatching(accessor, exp), + }; + } +} + +type NumberOfSnippetsProvider = (exp: TelemetryWithExp) => number; +// Add here for more values for other language ids. +const numberOfSnippets: ReadonlyMap<string, NumberOfSnippetsProvider> = new Map<string, NumberOfSnippetsProvider>([ + ['cpp', getCppNumberOfSnippets], +]); + +export function getNumberOfSnippets(exp: TelemetryWithExp, langId: string): number { + const provider: NumberOfSnippetsProvider | undefined = numberOfSnippets.get(langId); + return provider ? provider(exp) : DEFAULT_NUM_SNIPPETS; +} + +export function useSubsetMatching(accessor: ServicesAccessor, telemetryWithExp: TelemetryWithExp): boolean { + return ( + ((telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.UseSubsetMatching] as boolean) || + getConfig(accessor, ConfigKey.UseSubsetMatching)) ?? + false + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/similarFileOptionsProviderCpp.ts b/completions-sample-code/vscode-node/lib/src/experiments/similarFileOptionsProviderCpp.ts new file mode 100644 index 0000000..49fef2c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/similarFileOptionsProviderCpp.ts @@ -0,0 +1,20 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { defaultCppSimilarFilesOptions, SimilarFilesOptions } from '../../../prompt/src/snippetInclusion/similarFiles'; +import { TelemetryWithExp } from '../telemetry'; +import { useSubsetMatching } from './similarFileOptionsProvider'; + +export function getCppSimilarFilesOptions(accessor: ServicesAccessor, telemetryWithExp: TelemetryWithExp): SimilarFilesOptions { + return { + ...defaultCppSimilarFilesOptions, + useSubsetMatching: useSubsetMatching(accessor, telemetryWithExp), + }; +} + +export function getCppNumberOfSnippets(telemetryWithExp: TelemetryWithExp): number { + return defaultCppSimilarFilesOptions.maxTopSnippets; +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/telemetryNames.ts b/completions-sample-code/vscode-node/lib/src/experiments/telemetryNames.ts new file mode 100644 index 0000000..073fb7f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/telemetryNames.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export enum ExpServiceTelemetryNames { + // these are defined (but not exported) in the code for the tas client, currently here: + // https://github.com/microsoft/tas-client/blob/75f8895b15ef5696653cbee134ccae24477b0b94/vscode-tas-client/src/vscode-tas-client/VSCodeTasClient.ts#L67 + featuresTelemetryPropertyName = 'VSCode.ABExp.Features', +} diff --git a/completions-sample-code/vscode-node/lib/src/experiments/test/features.test.ts b/completions-sample-code/vscode-node/lib/src/experiments/test/features.test.ts new file mode 100644 index 0000000..301dd8b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/experiments/test/features.test.ts @@ -0,0 +1,51 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { extractRepoInfoInBackground } from '../../prompt/repository'; +import { TelemetryData } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { makeFsUri } from '../../util/uri'; +import { ICompletionsFeaturesService } from '../featuresService'; + +suite('updateExPValuesAndAssignments', function () { + let accessor: ServicesAccessor; + + const filenameUri = makeFsUri(__filename); + + setup(async function () { + accessor = createLibTestingContext().createTestingAccessor(); + // Trigger extractRepoInfoInBackground early + add a sleep to force repo info to be available + extractRepoInfoInBackground(accessor, filenameUri); + await new Promise(resolve => setTimeout(resolve, 100)); + }); + + test('If no options are provided, repo filters should be empty and there should be no telemetry properties or measurements', async function () { + const featuresService = accessor.get(ICompletionsFeaturesService); + const telemetry = await featuresService.updateExPValuesAndAssignments(); + + assert.deepStrictEqual(telemetry.properties, {}); + assert.deepStrictEqual(telemetry.measurements, {}); + + const filters = telemetry.filtersAndExp.filters.toHeaders(); + assert.deepStrictEqual(filters['X-Copilot-Repository'], undefined); + assert.deepStrictEqual(filters['X-Copilot-FileType'], undefined); + }); + + test('If telemetry data is passed as a parameter, it should be used in the resulting telemetry object', async function () { + const telemetryData = TelemetryData.createAndMarkAsIssued({ foo: 'bar' }, { baz: 42 }); + + const featuresService = accessor.get(ICompletionsFeaturesService); + const telemetry = await featuresService.updateExPValuesAndAssignments(undefined, telemetryData); + + assert.deepStrictEqual(telemetry.properties, { foo: 'bar' }); + assert.deepStrictEqual(telemetry.measurements, { baz: 42 }); + + const filters = telemetry.filtersAndExp.filters.toHeaders(); + assert.deepStrictEqual(filters['X-Copilot-Repository'], undefined); + assert.deepStrictEqual(filters['X-Copilot-FileType'], undefined); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/fileReader.ts b/completions-sample-code/vscode-node/lib/src/fileReader.ts new file mode 100644 index 0000000..b7d770a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/fileReader.ts @@ -0,0 +1,86 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { IInstantiationService } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFileSystemService } from './fileSystem'; +import { CopilotTextDocument, ITextDocument, TextDocumentIdentifier, TextDocumentResult } from './textDocument'; +import { ICompletionsTextDocumentManagerService } from './textDocumentManager'; +import { isDocumentValid } from './util/documentEvaluation'; +import { basename } from './util/uri'; + +export const ICompletionsFileReaderService = createServiceIdentifier<ICompletionsFileReaderService>('ICompletionsFileReaderService'); +export interface ICompletionsFileReaderService { + readonly _serviceBrand: undefined; + + getRelativePath(doc: TextDocumentIdentifier): string | undefined; + + getOrReadTextDocument(doc: TextDocumentIdentifier): Promise<TextDocumentResult>; + + getOrReadTextDocumentWithFakeClientProperties( + doc: TextDocumentIdentifier + ): Promise<TextDocumentResult<ITextDocument>>; +} + +export class FileReader implements ICompletionsFileReaderService { + declare _serviceBrand: undefined; + constructor( + @ICompletionsTextDocumentManagerService private readonly documentManagerService: ICompletionsTextDocumentManagerService, + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsFileSystemService private readonly fileSystemService: ICompletionsFileSystemService, + ) { } + + getRelativePath(doc: TextDocumentIdentifier) { + return this.documentManagerService.getRelativePath(doc) ?? basename(doc.uri); + } + + getOrReadTextDocument(doc: TextDocumentIdentifier): Promise<TextDocumentResult> { + return this.readFile(doc.uri); + } + + getOrReadTextDocumentWithFakeClientProperties( + doc: TextDocumentIdentifier + ): Promise<TextDocumentResult<ITextDocument>> { + return this.readFile(doc.uri); + } + + /** + * @deprecated use `getOrReadTextDocument` instead + */ + protected async readFile(uri: string): Promise<TextDocumentResult<ITextDocument>> { + const documentResult = await this.documentManagerService.getTextDocumentWithValidation({ uri }); + if (documentResult.status !== 'notfound') { + return documentResult; + } + try { + const fileSizeMB = await this.getFileSizeMB(uri); + // Note: the real production behavior actually blocks files larger than 5MB + if (fileSizeMB > 1) { + // Using notfound instead of invalid because of the mapping in statusFromTextDocumentResult + return { status: 'notfound' as const, message: 'File too large' }; + } + const text = await this.doReadFile(uri); + + // Note, that we check for blocked files even for empty files! + const rcmResult = await this.instantiationService.invokeFunction(isDocumentValid, { uri }); + if (rcmResult.status === 'valid') { + const doc = CopilotTextDocument.create(uri, 'UNKNOWN', -1, text); + return { status: 'valid' as const, document: doc }; + } + + return rcmResult; + } catch (e) { + return { status: 'notfound' as const, message: 'File not found' }; + } + } + + private async doReadFile(uri: string) { + return await this.fileSystemService.readFileString(uri); + } + + private async getFileSizeMB(uri: string) { + const stat = await this.fileSystemService.stat(uri); + return stat.size / 1024 / 1024; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/fileSystem.ts b/completions-sample-code/vscode-node/lib/src/fileSystem.ts new file mode 100644 index 0000000..15dbda2 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/fileSystem.ts @@ -0,0 +1,67 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../util/common/services'; + +/** + * `FileType` identifies the type of a file. `SymbolicLink` may be combined + * with other types, e.g. `FileType.Directory | FileType.SymbolicLink`. + */ +export enum FileType { + /** The file type is not known. */ + Unknown = 0, + /** The file is a regular file. */ + File = 1, + /** The file is a directory. */ + Directory = 2, + /** The file is a symbolic link. */ + SymbolicLink = 64, +} + +/** + * The `FileStat`-type represents metadata about a file + */ +export interface FileStat { + /** + * The creation timestamp in milliseconds elapsed since January 1, 1970 00:00:00 UTC. + */ + ctime: number; + + /** + * The modification timestamp in milliseconds elapsed since January 1, 1970 00:00:00 UTC. + * + * *Note:* If the file changed, it is important to provide an updated `mtime` that advanced + * from the previous value. Otherwise there may be optimizations in place that will not show + * the updated file contents in an editor for example. + */ + + mtime: number; + /** + * The size in bytes. + * + * *Note:* If the file changed, it is important to provide an updated `size`. Otherwise there + * may be optimizations in place that will not show the updated file contents in an editor for + * example. + */ + size: number; + /** + * The type of file. + * + * *Note:* This is a bit field. Multiple flags may be set on it, e.g. + * `FileType.File | FileType.SymbolicLink`. + */ + type: FileType; +} + +export type FileIdentifier = string | { readonly uri: string }; + +export const ICompletionsFileSystemService = createServiceIdentifier<ICompletionsFileSystemService>('ICompletionsFileSystemService'); +export interface ICompletionsFileSystemService { + readonly _serviceBrand: undefined; + + readFileString(uri: FileIdentifier): Promise<string>; + stat(uri: FileIdentifier): Promise<FileStat>; + readDirectory(uri: FileIdentifier): Promise<[string, FileType][]>; +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/asyncCompletions.ts b/completions-sample-code/vscode-node/lib/src/ghostText/asyncCompletions.ts new file mode 100644 index 0000000..c19b964 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/asyncCompletions.ts @@ -0,0 +1,309 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { CancellationTokenSource } from '../../../types/src'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { LRUCacheMap } from '../helpers/cache'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { APIChoice } from '../openai/openai'; +import { Prompt } from '../prompt/prompt'; +import { TelemetryWithExp } from '../telemetry'; +import { Deferred } from '../util/async'; +import { ReplaySubject } from '../util/subject'; +import { GetNetworkCompletionsType } from './ghostText'; + +enum AsyncCompletionRequestState { + Completed, + Error, + Pending, +} + +interface BaseAsyncCompletionRequest { + cancellationTokenSource: CancellationTokenSource; + headerRequestId: string; + partialCompletionText?: string; + prefix: string; + prompt: Prompt; + subject: ReplaySubject<AsyncCompletionRequest>; +} + +interface PendingAsyncCompletionRequest extends BaseAsyncCompletionRequest { + state: AsyncCompletionRequestState.Pending; +} + +interface CompletedAsyncCompletionRequest extends BaseAsyncCompletionRequest { + state: AsyncCompletionRequestState.Completed; + choice: APIChoice; + result: GetNetworkCompletionsType; + allChoicesPromise: Promise<void>; +} + +type AsyncCompletionRequest = PendingAsyncCompletionRequest | CompletedAsyncCompletionRequest; + +export const ICompletionsAsyncManagerService = createServiceIdentifier<ICompletionsAsyncManagerService>('ICompletionsAsyncManagerService'); +export interface ICompletionsAsyncManagerService { + readonly _serviceBrand: undefined; + clear(): void; + shouldWaitForAsyncCompletions(prefix: string, prompt: Prompt): boolean; + updateCompletion(headerRequestId: string, text: string): void; + queueCompletionRequest( + headerRequestId: string, + prefix: string, + prompt: Prompt, + cancellationTokenSource: CancellationTokenSource, + resultPromise: Promise<GetNetworkCompletionsType> + ): Promise<void>; + getFirstMatchingRequestWithTimeout( + headerRequestId: string, + prefix: string, + prompt: Prompt, + isSpeculative: boolean, + telemetryWithExp: TelemetryWithExp + ): Promise<[APIChoice, Promise<void>] | undefined>; + getFirstMatchingRequest( + headerRequestId: string, + prefix: string, + prompt: Prompt, + isSpeculative: boolean + ): Promise<[APIChoice, Promise<void>] | undefined>; +} + +export class AsyncCompletionManager implements ICompletionsAsyncManagerService { + declare _serviceBrand: undefined; + + #logger = new Logger('AsyncCompletionManager'); + + /** Mapping of headerRequestId to completion request */ + private readonly requests = new LRUCacheMap<string, AsyncCompletionRequest>(100); + + /** The most recently requested (either via getFirstMatchingRequest or + * getFirstMatchingRequestWithTimeout) header request ID. Serves as a lock + * for cancellation. Since we only want to cancel requests that don't match + * the most recent request prefix. */ + private mostRecentRequestId = ''; + + constructor( + @ICompletionsFeaturesService private readonly featuresService: ICompletionsFeaturesService, + @ICompletionsLogTargetService private readonly logTarget: ICompletionsLogTargetService, + ) { } + + clear() { + this.requests.clear(); + } + + /** + * Check if there are any candidate completions for the current position. + * We need to strike the right balance between queuing completions as the + * user types, without queuing one per keystroke. This method should return + * true if we don't have any completions that match the current position. + * This method should return false if we have reasonable candidates that + * match the current position. + */ + shouldWaitForAsyncCompletions(prefix: string, prompt: Prompt): boolean { + // TODO: Consider adding a minimum threshold for candidate completions, + // where we will queue more if the user's typing seems to be diverging + // from current speculation. + for (const [_, request] of this.requests) { + if (isCandidate(prefix, prompt, request)) { + return true; + } + } + return false; + } + + /** + * Called from a FinishedCallback to report partial results as a completion + * is streamed back from the server. + */ + updateCompletion(headerRequestId: string, text: string) { + const request = this.requests.get(headerRequestId); + if (request === undefined) { return; } + request.partialCompletionText = text; + request.subject.next(request); + } + + /** + * Adds an in-flight completion request to the requests map for tracking. + * Once the request is completed it is removed from the requests map. + */ + queueCompletionRequest( + headerRequestId: string, + prefix: string, + prompt: Prompt, + cancellationTokenSource: CancellationTokenSource, + resultPromise: Promise<GetNetworkCompletionsType> + ) { + this.#logger.debug(this.logTarget, + `[${headerRequestId}] Queueing async completion request:`, + prefix.substring(prefix.lastIndexOf('\n') + 1) + ); + const subject = new ReplaySubject<AsyncCompletionRequest>(); + this.requests.set(headerRequestId, { + state: AsyncCompletionRequestState.Pending, + cancellationTokenSource, + headerRequestId, + prefix, + prompt, + subject, + }); + return resultPromise + .then(result => { + this.requests.delete(headerRequestId); + if (result.type !== 'success') { + this.#logger.debug(this.logTarget, `[${headerRequestId}] Request failed with`, result.reason); + subject.error(result.reason); + return; + } + const completed: CompletedAsyncCompletionRequest = { + cancellationTokenSource, + headerRequestId, + prefix, + prompt, + subject, + choice: result.value[0], + result, + state: AsyncCompletionRequestState.Completed, + allChoicesPromise: result.value[1], + }; + this.requests.set(headerRequestId, completed); + subject.next(completed); + subject.complete(); + }) + .catch((e: unknown) => { + this.#logger.error(this.logTarget, `[${headerRequestId}] Request errored with`, e); + this.requests.delete(headerRequestId); + subject.error(e); + }); + } + + /** Returns the first matching completion or times out. */ + getFirstMatchingRequestWithTimeout( + headerRequestId: string, + prefix: string, + prompt: Prompt, + isSpeculative: boolean, + telemetryWithExp: TelemetryWithExp + ): Promise<[APIChoice, Promise<void>] | undefined> { + const timeout = this.featuresService.asyncCompletionsTimeout(telemetryWithExp); + if (timeout < 0) { + this.#logger.debug(this.logTarget, `[${headerRequestId}] Waiting for completions without timeout`); + return this.getFirstMatchingRequest(headerRequestId, prefix, prompt, isSpeculative); + } + this.#logger.debug(this.logTarget, `[${headerRequestId}] Waiting for completions with timeout of ${timeout}ms`); + return Promise.race([ + this.getFirstMatchingRequest(headerRequestId, prefix, prompt, isSpeculative), + new Promise<null>(r => setTimeout(() => r(null), timeout)), + ]).then(result => { + if (result === null) { + this.#logger.debug(this.logTarget, `[${headerRequestId}] Timed out waiting for completion`); + return undefined; + } + return result; + }); + } + + /** + * Returns the first resolved matching completion request. Modifies the + * returned APIChoice to match the current prompt. + */ + async getFirstMatchingRequest( + headerRequestId: string, + prefix: string, + prompt: Prompt, + isSpeculative: boolean + ): Promise<[APIChoice, Promise<void>] | undefined> { + if (!isSpeculative) { this.mostRecentRequestId = headerRequestId; } + let resolved = false; + const deferred = new Deferred<[APIChoice, Promise<void>] | undefined>(); + const subscriptions = new Map<string, () => void>(); + const finishRequest = (id: string) => () => { + const subscription = subscriptions.get(id); + if (subscription === undefined) { return; } + subscription(); + subscriptions.delete(id); + if (!resolved && subscriptions.size === 0) { + // TODO: Check for new candidates before resolving. + resolved = true; + this.#logger.debug(this.logTarget, `[${headerRequestId}] No matching completions found`); + deferred.resolve(undefined); + } + }; + const next = (request: AsyncCompletionRequest) => { + if (isCandidate(prefix, prompt, request)) { + if (request.state === AsyncCompletionRequestState.Completed) { + const remainingPrefix = prefix.substring(request.prefix.length); + let { completionText } = request.choice; + if ( + !completionText.startsWith(remainingPrefix) || + completionText.length <= remainingPrefix.length + ) { + finishRequest(request.headerRequestId)(); + return; + } + completionText = completionText.substring(remainingPrefix.length); + request.choice.telemetryData.measurements.foundOffset = remainingPrefix.length; + this.#logger.debug(this.logTarget, + `[${headerRequestId}] Found completion at offset ${remainingPrefix.length}: ${JSON.stringify(completionText)}` + ); + deferred.resolve([{ ...request.choice, completionText }, request.allChoicesPromise]); + resolved = true; + } + } else { + this.cancelRequest(headerRequestId, request); + finishRequest(request.headerRequestId)(); + } + }; + for (const [id, request] of this.requests) { + if (isCandidate(prefix, prompt, request)) { + subscriptions.set( + id, + request.subject.subscribe({ + next, + error: finishRequest(id), + complete: finishRequest(id), + }) + ); + } else { + this.cancelRequest(headerRequestId, request); + } + } + return deferred.promise.finally(() => { + for (const dispose of subscriptions.values()) { + dispose(); + } + }); + } + + /** + * Attempts to cancel a request if it is still pending and the request + * attempting the cancellation (that it no longer matches) is the most + * recent request. + * + * @param headerRequestId The request id for the call to + * getFirstMatchingRequest that the `request` no longer matches. + * @param request The request to cancel + */ + private cancelRequest(headerRequestId: string, request: AsyncCompletionRequest) { + if (headerRequestId !== this.mostRecentRequestId) { return; } + if (request.state === AsyncCompletionRequestState.Completed) { return; } + this.#logger.debug(this.logTarget, `[${headerRequestId}] Cancelling request: ${request.headerRequestId}`); + request.cancellationTokenSource.cancel(); + this.requests.delete(request.headerRequestId); + } +} + +function isCandidate(prefix: string, prompt: Prompt, request: AsyncCompletionRequest): boolean { + if (request.prompt.suffix !== prompt.suffix) { return false; } + if (!prefix.startsWith(request.prefix)) { return false; } + const remainingPrefix = prefix.substring(request.prefix.length); + if (request.state === AsyncCompletionRequestState.Completed) { + return ( + request.choice.completionText.startsWith(remainingPrefix) && + request.choice.completionText.trimEnd().length > remainingPrefix.length + ); + } + if (request.partialCompletionText === undefined) { return true; } + return request.partialCompletionText.startsWith(remainingPrefix); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/blockTrimmer.ts b/completions-sample-code/vscode-node/lib/src/ghostText/blockTrimmer.ts new file mode 100644 index 0000000..5242540 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/blockTrimmer.ts @@ -0,0 +1,328 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { StatementNode, StatementTree } from './statementTree'; +import { IPosition, TextDocumentContents } from '../textDocument'; + +/** + * BlockTrimmer base class. + */ +export abstract class BlockTrimmer { + static isSupported(languageId: string): boolean { + return StatementTree.isSupported(languageId); + } + + /** Tests for the subset of supported languages that are trimmed by default */ + static isTrimmedByDefault(languageId: string): boolean { + return StatementTree.isTrimmedByDefault(languageId); + } + + constructor( + protected readonly languageId: string, + protected readonly prefix: string, + protected readonly completion: string + ) { } + + abstract getCompletionTrimOffset(): Promise<number | undefined>; + + protected async withParsedStatementTree<T>(fn: (tree: StatementTree) => Promise<T> | T): Promise<T> { + const tree = StatementTree.create( + this.languageId, + this.prefix + this.completion, + this.prefix.length, + this.prefix.length + this.completion.length + ); + await tree.build(); + + try { + return await fn(tree); + } finally { + tree[Symbol.dispose](); + } + } + + protected trimmedCompletion(offset: number | undefined): string { + return offset === undefined ? this.completion : this.completion.substring(0, offset); + } + + /** + * Gets the statement at the cursor position. + * If the cursor is not within a statement (e.g. it's on an error node), + * returns the first statement from the tree (if any). + */ + protected getStatementAtCursor(tree: StatementTree): StatementNode | undefined { + return tree.statementAt(Math.max(this.prefix.length - 1, 0)) ?? tree.statements[0]; + } + + protected getContainingBlockOffset(stmt: StatementNode | undefined): number | undefined { + let trimTo: StatementNode | undefined; + if (stmt && this.isCompoundStatement(stmt)) { + // for compound statement types, trim to the current statement + trimTo = stmt; + } else if (stmt) { + // for non-compound statement types, trim to the closest compound ancestor + let parent = stmt.parent; + while (parent && !this.isCompoundStatement(parent)) { + parent = parent.parent; + } + trimTo = parent; + } + + if (trimTo) { + const newOffset = this.asCompletionOffset(trimTo.node.endIndex); + + // don't trim trailing whitespace as that will terminate the completion prematurely + if (newOffset && this.completion.substring(newOffset).trim() !== '') { return newOffset; } + } + return undefined; + } + + protected hasNonStatementContentAfter(stmt: StatementNode | undefined): boolean { + if (!stmt || !stmt.nextSibling) { return false; } + const spanStart = this.asCompletionOffset(stmt.node.endIndex); + const spanEnd = this.asCompletionOffset(stmt.nextSibling.node.startIndex); + const content = this.completion.substring(Math.max(0, spanStart ?? 0), Math.max(0, spanEnd ?? 0)); + return content.trim() !== ''; + } + + protected asCompletionOffset(offset: number | undefined): number | undefined { + return offset === undefined ? undefined : offset - this.prefix.length; + } + + protected isCompoundStatement(stmt: StatementNode): boolean { + return stmt.isCompoundStatementType || stmt.children.length > 0; + } +} + +/** + * A block trimmer that tries to obtain the longest reasonable completion + * within its line limit. This results in a more verbose completion. + * + * Don't delete it is used in tests. + */ +export class VerboseBlockTrimmer extends BlockTrimmer { + private readonly offsetLimit: number | undefined; + + constructor( + languageId: string, + prefix: string, + completion: string, + private readonly lineLimit: number = 10 + ) { + super(languageId, prefix, completion); + // determine the end of the lineLimit line as an offset into the completion + const completionLineEnds = [...this.completion.matchAll(/\n/g)]; + if (completionLineEnds.length >= this.lineLimit && this.lineLimit > 0) { + this.offsetLimit = completionLineEnds[this.lineLimit - 1].index; + } else { + this.offsetLimit = undefined; + } + } + + async getCompletionTrimOffset(): Promise<number | undefined> { + return await this.withParsedStatementTree(tree => { + const stmt = this.getStatementAtCursor(tree); + + // do not go past the containing block + let offset = this.getContainingBlockOffset(stmt); + + // first try trimming at a blank line + if (!this.isWithinLimit(offset)) { + offset = this.trimToBlankLine(offset); + } + + // then try trimming at a statement + if (!this.isWithinLimit(offset)) { + offset = this.trimToStatement(stmt, offset); + } + + return offset; + }); + } + + private isWithinLimit(offset: number | undefined): boolean { + return this.offsetLimit === undefined || (offset !== undefined && offset <= this.offsetLimit); + } + + private trimToBlankLine(offset: number | undefined): number | undefined { + const blankLines = [...this.trimmedCompletion(offset).matchAll(/\r?\n\s*\r?\n/g)].reverse(); + while (blankLines.length > 0 && !this.isWithinLimit(offset)) { + const match = blankLines.pop()!; + offset = match.index; + } + return offset; + } + + private trimToStatement(stmt: StatementNode | undefined, offset: number | undefined): number | undefined { + const min = this.prefix.length; + const max = this.prefix.length + (this.offsetLimit ?? this.completion.length); + let s = stmt; + let next = stmt?.nextSibling; + while (next && next.node.endIndex <= max && !this.hasNonStatementContentAfter(s)) { + s = next; + next = next.nextSibling; + } + if (s && s === stmt && s.node.endIndex <= min) { + s = next; + } + if (s && s.node.endIndex > max) { + // break at an internal statement if possible + return this.trimToStatement(s.children[0], this.asCompletionOffset(s.node.endIndex)); + } + return this.asCompletionOffset(s?.node?.endIndex) ?? offset; + } +} + +/** + * A block trimmer that stops when it's likely the end of a logical section has + * been reached, such as the start of a new compound statement. This results in + * a more terse completion. + */ +export class TerseBlockTrimmer extends BlockTrimmer { + private readonly limitOffset: number | undefined; + private readonly lookAheadOffset: number | undefined; + + constructor( + languageId: string, + prefix: string, + completion: string, + private readonly lineLimit: number = 3, + private readonly lookAhead: number = 7 + ) { + super(languageId, prefix, completion); + // determine the end of the lineLimit line as an offset into the completion + const completionLineEnds = [...this.completion.matchAll(/\n/g)]; + const limitAndLookAhead = this.lineLimit + this.lookAhead; + if (completionLineEnds.length >= this.lineLimit && this.lineLimit > 0) { + this.limitOffset = completionLineEnds[this.lineLimit - 1].index; + } + if (completionLineEnds.length >= limitAndLookAhead && limitAndLookAhead > 0) { + this.lookAheadOffset = completionLineEnds[limitAndLookAhead - 1].index; + } + } + + async getCompletionTrimOffset(): Promise<number | undefined> { + return await this.withParsedStatementTree(tree => { + const stmt = tree.statementAt(this.stmtStartPos()); + + // do not go past the containing block + let offset = this.getContainingBlockOffset(stmt); + + // trim at any blank lines + offset = this.trimAtFirstBlankLine(offset); + + // trim at new blocks starts or areas of comments + if (stmt) { + offset = this.trimAtStatementChange(stmt, offset); + } + + // hard trim at the line limit if we have enough context + if (this.limitOffset && this.lookAheadOffset && (offset === undefined || offset > this.lookAheadOffset)) { + return this.limitOffset; + } + + return offset; + }); + } + + /** + * Return the position of the first non-whitespace character to the right + * of the cursor, or the start of the completion if it is blank. + */ + private stmtStartPos(): number { + const match = this.completion.match(/\S/); + if (match && match.index !== undefined) { + return this.prefix.length + match.index; + } + return Math.max(this.prefix.length - 1, 0); + } + + private trimAtFirstBlankLine(offset: number | undefined): number | undefined { + const blankLines = [...this.trimmedCompletion(offset).matchAll(/\r?\n\s*\r?\n/g)]; + + while (blankLines.length > 0 && (offset === undefined || offset > blankLines[0].index)) { + const match = blankLines.shift()!; + if (this.completion.substring(0, match.index).trim() !== '') { + return match.index; + } + } + return offset; + } + + private trimAtStatementChange(stmt: StatementNode, offset: number | undefined): number | undefined { + const min = this.prefix.length; + const max = this.prefix.length + (offset ?? this.completion.length); + + // if the first statement is a compound statement, trim to the first statement + if (stmt.node.endIndex > min && this.isCompoundStatement(stmt)) { + // if we have a next sibling, the statement is likely finished + if (stmt.nextSibling && stmt.node.endIndex < max) { + return this.asCompletionOffset(stmt.node.endIndex); + } + return offset; + } + + // otherwise, stop at the first compound statement or non-statement content + let s = stmt; + let next = stmt.nextSibling; + while ( + next && + next.node.endIndex <= max && + !this.hasNonStatementContentAfter(s) && + !this.isCompoundStatement(next) + ) { + s = next; + next = next.nextSibling; + } + if (next && s.node.endIndex > min && s.node.endIndex < max) { + return this.asCompletionOffset(s.node.endIndex); + } + return offset; + } +} + +export enum BlockPositionType { + NonBlock = 'non-block', + EmptyBlock = 'empty-block', + BlockEnd = 'block-end', + MidBlock = 'mid-block', +} + +export async function getBlockPositionType( + document: TextDocumentContents, + position: IPosition +): Promise<BlockPositionType> { + const text = document.getText(); + const offset = document.offsetAt(position); + const tree = StatementTree.create(document.detectedLanguageId, text, 0, text.length); + try { + await tree.build(); + + const stmt = tree.statementAt(offset); + + if (!stmt) { return BlockPositionType.NonBlock; } + + if (!stmt.isCompoundStatementType && stmt.children.length === 0) { + if (stmt.parent && !stmt.nextSibling && stmt.node.endPosition.row <= position.line) { + return BlockPositionType.BlockEnd; + } else if (stmt.parent) { + return BlockPositionType.MidBlock; + } + return BlockPositionType.NonBlock; + } + + if (stmt.children.length === 0) { + return BlockPositionType.EmptyBlock; + } + + const lastChild = stmt.children[stmt.children.length - 1]; + if (offset < lastChild.node.startIndex) { + return BlockPositionType.MidBlock; + } + + return BlockPositionType.BlockEnd; + } finally { + tree[Symbol.dispose](); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/completionsCache.ts b/completions-sample-code/vscode-node/lib/src/ghostText/completionsCache.ts new file mode 100644 index 0000000..8ef5cb7 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/completionsCache.ts @@ -0,0 +1,69 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { LRURadixTrie } from '../helpers/radix'; +import { APIChoice } from '../openai/openai'; + +interface CompletionsCacheContents { + content: { + suffix: string; + choice: APIChoice; + }[]; +} + +export const ICompletionsCacheService = createServiceIdentifier<ICompletionsCacheService>('ICompletionsCacheService'); +export interface ICompletionsCacheService { + readonly _serviceBrand: undefined; + + /** Given a document prefix and suffix, return all of the completions that match. */ + findAll(prefix: string, suffix: string): APIChoice[]; + + /** Add cached completions for a given prefix. */ + append(prefix: string, suffix: string, choice: APIChoice): void; + + clear(): void; +} + +/** Caches recent completions by document prefix. */ +export class CompletionsCache implements ICompletionsCacheService { + readonly _serviceBrand: undefined; + + private cache = new LRURadixTrie<CompletionsCacheContents>(100); + + /** Given a document prefix and suffix, return all of the completions that match. */ + findAll(prefix: string, suffix: string): APIChoice[] { + return this.cache.findAll(prefix).flatMap(({ remainingKey, value }) => + value.content + .filter( + c => + c.suffix === suffix && + c.choice.completionText.startsWith(remainingKey) && + c.choice.completionText.length > remainingKey.length + ) + .map(c => ({ + ...c.choice, + completionText: c.choice.completionText.slice(remainingKey.length), + telemetryData: c.choice.telemetryData.extendedBy({}, { foundOffset: remainingKey.length }), + })) + ); + } + + /** Add cached completions for a given prefix. */ + append(prefix: string, suffix: string, choice: APIChoice) { + const existing = this.cache.findAll(prefix); + // Append to an existing array if there is an exact match. + if (existing.length > 0 && existing[0].remainingKey === '') { + const content = existing[0].value.content; + this.cache.set(prefix, { content: [...content, { suffix, choice }] }); + } else { + // Otherwise, add a new value. + this.cache.set(prefix, { content: [{ suffix, choice }] }); + } + } + + clear() { + this.cache = new LRURadixTrie<CompletionsCacheContents>(100); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/configBlockMode.ts b/completions-sample-code/vscode-node/lib/src/ghostText/configBlockMode.ts new file mode 100644 index 0000000..5638ce6 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/configBlockMode.ts @@ -0,0 +1,74 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// The following code was moved from config.ts into here to break the cyclic dependencies + +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { BlockMode } from '../../../../../completions/common/config'; +import { isSupportedLanguageId } from '../../../prompt/src/parse'; +import { ConfigKey, getConfig } from '../config'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { TelemetryWithExp } from '../telemetry'; +import { BlockTrimmer } from './blockTrimmer'; +import { StatementTree } from './statementTree'; + +export const ICompletionsBlockModeConfig = createServiceIdentifier<ICompletionsBlockModeConfig>('ICompletionsBlockModeConfig'); +export interface ICompletionsBlockModeConfig { + readonly _serviceBrand: undefined; + forLanguage(languageId: string, telemetryData: TelemetryWithExp): BlockMode; +} + +export class ConfigBlockModeConfig implements ICompletionsBlockModeConfig { + declare _serviceBrand: undefined; + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsFeaturesService private readonly featuresService: ICompletionsFeaturesService, + ) { } + + forLanguage(languageId: string, telemetryData: TelemetryWithExp): BlockMode { + const overrideBlockMode = this.featuresService.overrideBlockMode(telemetryData); + if (overrideBlockMode) { + return toApplicableBlockMode(overrideBlockMode, languageId); + } + const progressiveReveal = this.featuresService.enableProgressiveReveal(telemetryData); + const config = this.instantiationService.invokeFunction(getConfig, ConfigKey.AlwaysRequestMultiline); + if (config ?? progressiveReveal) { + return toApplicableBlockMode(BlockMode.MoreMultiline, languageId); + } + + if (BlockTrimmer.isTrimmedByDefault(languageId)) { + return toApplicableBlockMode(BlockMode.MoreMultiline, languageId); + } + // special casing once cancellations based on tree-sitter propagate to + // the proxy. + if (languageId === 'ruby') { + return BlockMode.Parsing; + } + // For existing multiline languages use standard tree-sitter based parsing + // plus proxy-side trimming + if (isSupportedLanguageId(languageId)) { + return BlockMode.ParsingAndServer; + } + return BlockMode.Server; + } +} + +function blockModeRequiresTreeSitter(blockMode: BlockMode): boolean { + return [BlockMode.Parsing, BlockMode.ParsingAndServer, BlockMode.MoreMultiline].includes(blockMode); +} + +/** + * Prevents tree-sitter parsing from being applied to languages we don't include + * parsers for. + */ +function toApplicableBlockMode(blockMode: BlockMode, languageId: string): BlockMode { + if (blockMode === BlockMode.MoreMultiline && StatementTree.isSupported(languageId)) { + return blockMode; + } + if (blockModeRequiresTreeSitter(blockMode) && !isSupportedLanguageId(languageId)) { + return BlockMode.Server; + } + return blockMode; +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/contextualFilterConstants.ts b/completions-sample-code/vscode-node/lib/src/ghostText/contextualFilterConstants.ts new file mode 100644 index 0000000..aea98b8 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/contextualFilterConstants.ts @@ -0,0 +1,101 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export const contextualFilterCharacterMap: { [key: string]: number } = { + ' ': 1, + '!': 2, + '"': 3, + '#': 4, + $: 5, + '%': 6, + '&': 7, + '\'': 8, + '(': 9, + ')': 10, + '*': 11, + '+': 12, + ',': 13, + '-': 14, + '.': 15, + '/': 16, + '0': 17, + '1': 18, + '2': 19, + '3': 20, + '4': 21, + '5': 22, + '6': 23, + '7': 24, + '8': 25, + '9': 26, + ':': 27, + ';': 28, + '<': 29, + '=': 30, + '>': 31, + '?': 32, + '@': 33, + A: 34, + B: 35, + C: 36, + D: 37, + E: 38, + F: 39, + G: 40, + H: 41, + I: 42, + J: 43, + K: 44, + L: 45, + M: 46, + N: 47, + O: 48, + P: 49, + Q: 50, + R: 51, + S: 52, + T: 53, + U: 54, + V: 55, + W: 56, + X: 57, + Y: 58, + Z: 59, + '[': 60, + '\\': 61, + ']': 62, + '^': 63, + _: 64, + '`': 65, + a: 66, + b: 67, + c: 68, + d: 69, + e: 70, + f: 71, + g: 72, + h: 73, + i: 74, + j: 75, + k: 76, + l: 77, + m: 78, + n: 79, + o: 80, + p: 81, + q: 82, + r: 83, + s: 84, + t: 85, + u: 86, + v: 87, + w: 88, + x: 89, + y: 90, + z: 91, + '{': 92, + '|': 93, + '}': 94, + '~': 95, +}; diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/copilotCompletion.ts b/completions-sample-code/vscode-node/lib/src/ghostText/copilotCompletion.ts new file mode 100644 index 0000000..48bcd06 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/copilotCompletion.ts @@ -0,0 +1,86 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; +import { TelemetryWithExp } from '../telemetry'; +import { IPosition, IRange, LocationFactory, TextDocumentContents } from '../textDocument'; +import { CompletionResult, ResultType } from './ghostText'; +import { ITextEditorOptions, normalizeIndentCharacter } from './normalizeIndent'; + +export interface CopilotCompletion { + uuid: string; + insertText: string; + range: IRange; + uri: string; + telemetry: TelemetryWithExp; + displayText: string; + position: IPosition; + offset: number; + index: number; + resultType: ResultType; + copilotAnnotations?: CopilotNamedAnnotationList; + clientCompletionId: string; +} + +export function completionsFromGhostTextResults( + completionResults: CompletionResult[], + resultType: ResultType, + document: TextDocumentContents, + position: IPosition, + textEditorOptions?: ITextEditorOptions, + lastShownCompletionIndex?: number +): CopilotCompletion[] { + const currentLine = document.lineAt(position); + let completions = completionResults.map(result => { + const range = LocationFactory.range( + LocationFactory.position(position.line, 0), + LocationFactory.position(position.line, position.character + result.suffixCoverage) + ); + let insertText = ''; + if (textEditorOptions) { + result.completion = normalizeIndentCharacter( + textEditorOptions, + result.completion, + currentLine.isEmptyOrWhitespace + ); + } + if ( + currentLine.isEmptyOrWhitespace && + (result.completion.displayNeedsWsOffset || // Deindenting case + // This enables stable behavior for deleting whitespace on blank lines + result.completion.completionText.startsWith(currentLine.text)) + ) { + insertText = result.completion.completionText; + } else { + const rangeFromStart = LocationFactory.range(range.start, position); + insertText = document.getText(rangeFromStart) + result.completion.displayText; + } + + const completion: CopilotCompletion = { + uuid: generateUuid(), + insertText, + range, + uri: document.uri, + index: result.completion.completionIndex, + telemetry: result.telemetry, + displayText: result.completion.displayText, + position, + offset: document.offsetAt(position), + resultType, + copilotAnnotations: result.copilotAnnotations, + clientCompletionId: result.clientCompletionId, + }; + return completion; + }); + //If we are in typing as suggested flow, we want to put the last displayed completion at the top of the list to keep it selected + if (resultType === ResultType.TypingAsSuggested && lastShownCompletionIndex !== undefined) { + const lastShownCompletion = completions.find(predicate => predicate.index === lastShownCompletionIndex); + if (lastShownCompletion) { + const restCompletions = completions.filter(predicate => predicate.index !== lastShownCompletionIndex); + completions = [lastShownCompletion, ...restCompletions]; + } + } + return completions; +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/current.ts b/completions-sample-code/vscode-node/lib/src/ghostText/current.ts new file mode 100644 index 0000000..8604193 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/current.ts @@ -0,0 +1,114 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { APIChoice } from '../openai/openai'; +import { ResultType } from './ghostText'; + +export const ICompletionsCurrentGhostText = createServiceIdentifier<ICompletionsCurrentGhostText>('ICompletionsCurrentGhostText'); +export interface ICompletionsCurrentGhostText { + readonly _serviceBrand: undefined; + + readonly clientCompletionId: string | undefined; + + currentRequestId: string | undefined; + + setGhostText(prefix: string, suffix: string, choices: APIChoice[], resultType: ResultType): void; + getCompletionsForUserTyping(prefix: string, suffix: string): APIChoice[] | undefined; + hasAcceptedCurrentCompletion(prefix: string, suffix: string): boolean; +} + +/** + * Stores the internal concept of the currently shown completion, as inferred by + * the output of getGhostText. Used to check if a subsequent call to + * getGhostText is typing-as-suggested. + */ +export class CurrentGhostText implements ICompletionsCurrentGhostText { + declare _serviceBrand: undefined; + /** The document prefix at the start of the typing-as-suggested flow. This + * does not use the prompt prefix since the ellision means that prefix is a + * sliding window over long documents. */ + private prefix?: string; + + /** The prompt suffix at the start of the typing-as-suggested flow. */ + private suffix?: string; + + /** The original APIChoice array created at the start of the + * typing-as-suggested flow. The first element in the array should be the + * completion shown to the user. */ + private choices: APIChoice[] = []; + + /** The currently shown completion id. */ + get clientCompletionId(): string | undefined { + return this.choices[0]?.clientCompletionId; + } + + /** The most recent inline completion request id, excluding speculative requests. */ + currentRequestId: string | undefined; + + /** Updates the current ghost text if it was not produced via + * TypingAsSuggested. Should only be called from the end of getGhostText. */ + setGhostText(prefix: string, suffix: string, choices: APIChoice[], resultType: ResultType) { + if (resultType === ResultType.TypingAsSuggested) { return; } + this.prefix = prefix; + this.suffix = suffix; + this.choices = choices; + } + + /** Returns the current choices if the request context matches. */ + getCompletionsForUserTyping(prefix: string, suffix: string): APIChoice[] | undefined { + const remainingPrefix = this.getRemainingPrefix(prefix, suffix); + if (remainingPrefix === undefined) { return; } + // If the first choice text does not match return empty to fall through + // to either the cache or network. + if (!startsWithAndExceeds(this.choices[0].completionText, remainingPrefix)) { return; } + return adjustChoicesStart(this.choices, remainingPrefix); + } + + /** Returns whether the current completion is fully completed, and covers a full line. */ + hasAcceptedCurrentCompletion(prefix: string, suffix: string): boolean { + const remainingPrefix = this.getRemainingPrefix(prefix, suffix); + if (remainingPrefix === undefined) { return false; } + + // Check if the completion text matches exactly + const exactMatch = remainingPrefix === this.choices?.[0].completionText; + + // Check finishReason - return false if it indicates that the server cut off a part of it (thus it might not complete a full line), due to RAI or snippy + const finishReason = this.choices?.[0].finishReason; + return exactMatch && finishReason === 'stop'; + } + + /** If the given document prefix and prompt suffix match the current + * completion returns the remaining prefix of the document after the stored + * prefix. Returns undefined if the completion does not match. */ + private getRemainingPrefix(prefix: string, suffix: string): string | undefined { + // Check that there is a current completion. + if (this.prefix === undefined || this.suffix === undefined || this.choices.length === 0) { return; } + // Check that the prompt suffixes are an exact match. + if (this.suffix !== suffix) { return; } + // Check that the document prefix is a prefix of the new prefix. + // This doesn't use the prompt prefix since the ellision means that + // subsequent prefixes will not be a prefix of earlier ones. + if (!prefix.startsWith(this.prefix)) { return; } + // Return the remaining new document prefix after the prefix stored for + // the current completion. + return prefix.substring(this.prefix.length); + } +} + +/** Returns choices adjusted to remove the remainingPrefix from the start of the + * completionText if it matches. */ +function adjustChoicesStart(choices: APIChoice[], remainingPrefix: string): APIChoice[] { + return choices + .filter(choice => startsWithAndExceeds(choice.completionText, remainingPrefix)) + .map(choice => ({ + ...choice, + completionText: choice.completionText.substring(remainingPrefix.length), + })); +} + +/** Returns true if `prefix` is a prefix of `text` and `text` is longer. */ +function startsWithAndExceeds(text: string, prefix: string) { + return text.startsWith(prefix) && text.length > prefix.length; +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/ghostText.ts b/completions-sample-code/vscode-node/lib/src/ghostText/ghostText.ts new file mode 100644 index 0000000..ae01f6a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/ghostText.ts @@ -0,0 +1,1573 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { ConfigKey as ChatConfigKey, IConfigurationService } from '../../../../../../platform/configuration/common/configurationService'; +import { IExperimentationService } from '../../../../../../platform/telemetry/common/nullExperimentationService'; +import { ITelemetryService } from '../../../../../../platform/telemetry/common/telemetry'; +import { createSha256Hash } from '../../../../../../util/common/crypto'; +import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { isSupportedLanguageId } from '../../../prompt/src/parse'; +import { initializeTokenizers } from '../../../prompt/src/tokenization'; +import { CancellationTokenSource, CancellationToken as ICancellationToken } from '../../../types/src'; +import { ICompletionsNotifierService } from '../completionNotifier'; +import { CompletionState } from '../completionState'; +import { BlockMode, ConfigKey, getConfig, shouldDoServerTrimming } from '../config'; +import { ICompletionsUserErrorNotifierService } from '../error/userErrorNotifier'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { isAbortError } from '../networking'; +import { EngineRequestInfo, getEngineRequestInfo } from '../openai/config'; +import { + CompletionHeaders, + CompletionRequestExtra, + CopilotUiKind, + FinishedCallback, + ICompletionsOpenAIFetcherService, + PostOptions +} from '../openai/fetch'; +import { APIChoice, getTemperatureForSamples } from '../openai/openai'; +import { ICompletionsStatusReporter } from '../progress'; +import { ICompletionsContextProviderBridgeService } from '../prompt/components/contextProviderBridge'; +import { ICompletionsContextProviderService } from '../prompt/contextProviderStatistics'; +import { + ContextIndentation, + contextIndentation, + isEmptyBlockStartUtil, + parsingBlockFinished, +} from '../prompt/parseBlock'; +import { ExtractPromptOptions, Prompt, PromptResponsePresent, extractPrompt, trimLastLine } from '../prompt/prompt'; +import { ComputationStatus, MaybeRepoInfo, extractRepoInfoInBackground } from '../prompt/repository'; +import { checkSuffix, postProcessChoiceInContext } from '../suggestions/suggestions'; +import { + TelemetryData, + TelemetryMeasurements, + TelemetryProperties, + TelemetryWithExp, + now, + telemetrizePromptLength, + telemetry, +} from '../telemetry'; +import { IPosition, LocationFactory, TextDocumentContents } from '../textDocument'; +import { delay } from '../util/async'; +import { ICompletionsRuntimeModeService } from '../util/runtimeMode'; +import { ICompletionsAsyncManagerService } from './asyncCompletions'; +import { BlockPositionType, BlockTrimmer, getBlockPositionType } from './blockTrimmer'; +import { ICompletionsCacheService } from './completionsCache'; +import { ICompletionsBlockModeConfig } from './configBlockMode'; +import { ICompletionsCurrentGhostText } from './current'; +import { requestMultilineScore } from './multilineModel'; +import { StreamedCompletionSplitter } from './streamedCompletionSplitter'; +import { + GhostTextResultWithTelemetry, + mkBasicResultTelemetry, + mkCanceledResultTelemetry, + resultTypeToString, +} from './telemetry'; + +const ghostTextLogger = new Logger('ghostText'); + +export interface GhostCompletion { + completionIndex: number; + completionText: string; + displayText: string; + displayNeedsWsOffset: boolean; +} + +export interface CompletionResult { + completion: GhostCompletion; + telemetry: TelemetryWithExp; + isMiddleOfTheLine: boolean; + suffixCoverage: number; + copilotAnnotations?: CopilotNamedAnnotationList; + clientCompletionId: string; +} + +export enum ResultType { + Network, + Cache, + TypingAsSuggested, + Cycling, + Async, +} + +// p50 line length is 19 characters (p95 is 73) +// average token length is around 4 characters +// the below values have quite a bit of buffer while bringing the limit in significantly from 500 +const maxSinglelineTokens = 20; + +export type GetNetworkCompletionsType = GhostTextResultWithTelemetry<[APIChoice, Promise<void>]>; + +type GetAllNetworkCompletionsType = GhostTextResultWithTelemetry<[APIChoice[], Promise<void>]>; + +export class CompletionsFromNetwork { + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsOpenAIFetcherService private readonly fetcherService: ICompletionsOpenAIFetcherService, + @ICompletionsFeaturesService private readonly featuresService: ICompletionsFeaturesService, + @ICompletionsRuntimeModeService private readonly runtimeMode: ICompletionsRuntimeModeService, + @ICompletionsLogTargetService private readonly logTarget: ICompletionsLogTargetService, + @ICompletionsCacheService private readonly completionsCacheService: ICompletionsCacheService, + @ICompletionsUserErrorNotifierService private readonly userErrorNotifier: ICompletionsUserErrorNotifierService, + @IConfigurationService private readonly configurationService: IConfigurationService, + @IExperimentationService private readonly expService: IExperimentationService, + ) { } + + /** Requests new completion from OpenAI, should be called if and only if the completions for given prompt were not cached before. + * It returns only first completion, additional completions are added to the caches in the background. + * Copies from the base telemetry data are used as the basis for each choice's telemetry. + */ + public async getCompletionsFromNetwork( + requestContext: RequestContext, + baseTelemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined, + finishedCb: FinishedCallback + ): Promise<GetNetworkCompletionsType> { + return this.genericGetCompletionsFromNetwork( + requestContext, + baseTelemetryData, + cancellationToken, + finishedCb, + 'completions', + async (requestStart, processingTime, choicesStream): Promise<GetNetworkCompletionsType> => { + const choicesIterator = choicesStream[Symbol.asyncIterator](); + + const firstRes = await choicesIterator.next(); + + if (firstRes.done) { + ghostTextLogger.debug(this.logTarget, 'All choices redacted'); + return { + type: 'empty', + reason: 'all choices redacted', + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + }; + } + if (cancellationToken?.isCancellationRequested) { + ghostTextLogger.debug(this.logTarget, 'Cancelled after awaiting redactedChoices iterator'); + return { + type: 'canceled', + reason: 'after awaiting redactedChoices iterator', + telemetryData: mkCanceledResultTelemetry(baseTelemetryData), + }; + } + + const firstChoice: APIChoice = firstRes.value; + + if (firstChoice === undefined) { + // This is probably unreachable given the firstRes.done check above + ghostTextLogger.debug(this.logTarget, 'Got undefined choice from redactedChoices iterator'); + return { + type: 'empty', + reason: 'got undefined choice from redactedChoices iterator', + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + }; + } + + this.instantiationService.invokeFunction(telemetryPerformance, 'performance', firstChoice, requestStart, processingTime); + + ghostTextLogger.debug(this.logTarget, `Awaited first result, id: ${firstChoice.choiceIndex}`); + // Adds first result to cache + const processedFirstChoice = postProcessChoices(firstChoice); + if (processedFirstChoice) { + appendToCache(this.completionsCacheService, requestContext, processedFirstChoice); + ghostTextLogger.debug(this.logTarget, + `GhostText first completion (index ${processedFirstChoice?.choiceIndex}): ${JSON.stringify(processedFirstChoice?.completionText)}` + ); + } + //Create promise for each result, don't `await` it (unless in test mode) but handle asynchronously with `.then()` + const cacheDone = (async () => { + const apiChoices: APIChoice[] = processedFirstChoice !== undefined ? [processedFirstChoice] : []; + for await (const choice of choicesStream) { + if (choice === undefined) { continue; } + ghostTextLogger.debug(this.logTarget, + `GhostText later completion (index ${choice?.choiceIndex}): ${JSON.stringify(choice.completionText)}` + ); + const processedChoice = postProcessChoices(choice, apiChoices); + if (!processedChoice) { continue; } + apiChoices.push(processedChoice); + appendToCache(this.completionsCacheService, requestContext, processedChoice); + } + })(); + if (this.runtimeMode.isRunningInTest()) { + await cacheDone; + } + if (processedFirstChoice) { + // Because we ask the server to stop at \n above, we don't need to force single line here + return { + type: 'success', + value: [makeGhostAPIChoice(processedFirstChoice, { forceSingleLine: false }), cacheDone], + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + telemetryBlob: baseTelemetryData, + resultType: ResultType.Network, + }; + } else { + return { + type: 'empty', + reason: 'got undefined processedFirstChoice', + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + }; + } + } + ); + } + + /** Requests new completion from OpenAI, should be called if and only if we are in the servers-side termination mode, and it's follow-up cycling request + * It returns all requested completions + * Copies from the base telemetry data are used as the basis for each choice's telemetry. + */ + public async getAllCompletionsFromNetwork( + requestContext: RequestContext, + baseTelemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined, + finishedCb: FinishedCallback + ): Promise<GetAllNetworkCompletionsType> { + return this.genericGetCompletionsFromNetwork( + requestContext, + baseTelemetryData, + cancellationToken, + finishedCb, + 'all completions', + async (requestStart, processingTime, choicesStream): Promise<GetAllNetworkCompletionsType> => { + const apiChoices: APIChoice[] = []; + for await (const choice of choicesStream) { + if (cancellationToken?.isCancellationRequested) { + ghostTextLogger.debug(this.logTarget, 'Cancelled after awaiting choices iterator'); + return { + type: 'canceled', + reason: 'after awaiting choices iterator', + telemetryData: mkCanceledResultTelemetry(baseTelemetryData), + }; + } + const processedChoice = postProcessChoices(choice, apiChoices); + if (!processedChoice) { continue; } + apiChoices.push(processedChoice); + } + //Append results to current completions cache, and network cache + if (apiChoices.length > 0) { + for (const choice of apiChoices) { + appendToCache(this.completionsCacheService, requestContext, choice); + } + + this.instantiationService.invokeFunction(telemetryPerformance, 'cyclingPerformance', apiChoices[0], requestStart, processingTime); + } + return { + type: 'success', + value: [apiChoices, Promise.resolve()], + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + telemetryBlob: baseTelemetryData, + resultType: ResultType.Cycling, + }; + } + ); + } + + private async genericGetCompletionsFromNetwork<T>( + requestContext: RequestContext, + baseTelemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined, + finishedCb: FinishedCallback, + what: string, + processChoices: ( + requestStart: number, + processingTime: number, + choicesStream: AsyncIterable<APIChoice> + ) => Promise<GhostTextResultWithTelemetry<T>> + ): Promise<GhostTextResultWithTelemetry<T>> { + ghostTextLogger.debug(this.logTarget, `Getting ${what} from network`); + + // copy the base telemetry data + baseTelemetryData = baseTelemetryData.extendedBy(); + + // Request one choice for automatic requests, three for invoked (cycling) requests. + const n = requestContext.isCycling ? 3 : 1; + const temperature = getTemperatureForSamples(this.runtimeMode, n); + const extra: CompletionRequestExtra = { + language: requestContext.languageId, + next_indent: requestContext.indentation.next ?? 0, + trim_by_indentation: shouldDoServerTrimming(requestContext.blockMode), + prompt_tokens: requestContext.prompt.prefixTokens ?? 0, + suffix_tokens: requestContext.prompt.suffixTokens ?? 0, + }; + const postOptions: PostOptions = { n, temperature, code_annotations: false }; + const modelTerminatesSingleline = + this.featuresService.modelAlwaysTerminatesSingleline(baseTelemetryData); + const simulateSingleline = + requestContext.blockMode === BlockMode.MoreMultiline && + BlockTrimmer.isSupported(requestContext.languageId) && + !modelTerminatesSingleline; + if (!requestContext.multiline && !simulateSingleline) { + // If we are not in multiline mode, we get the server to truncate the results. This does mean that we + // also cache a single line result which will be reused even if we are later in multiline mode. This is + // an acceptable trade-off as the transition should be relatively rare and truncating on the server is + // more efficient. + // Note that this also means we don't need to truncate when creating the GhostAPIChoice object below. + postOptions['stop'] = ['\n']; + } else if (requestContext.stop) { + postOptions['stop'] = requestContext.stop; + } + if (requestContext.maxTokens !== undefined) { + postOptions['max_tokens'] = requestContext.maxTokens; + } + + const requestStart = Date.now(); + + // extend telemetry data + const newProperties: { [key: string]: string } = { + endpoint: 'completions', + uiKind: CopilotUiKind.GhostText, + temperature: JSON.stringify(temperature), + n: JSON.stringify(n), + stop: JSON.stringify(postOptions['stop']) ?? 'unset', + logit_bias: JSON.stringify(null), + }; + + Object.assign(baseTelemetryData.properties, newProperties); + + try { + const completionParams = { + prompt: requestContext.prompt, + languageId: requestContext.languageId, + repoInfo: requestContext.repoInfo, + ourRequestId: requestContext.ourRequestId, + engineModelId: requestContext.engineModelId, + count: n, + uiKind: CopilotUiKind.GhostText, + postOptions, + headers: requestContext.headers, + extra, + }; + const res = + this.configurationService.getExperimentBasedConfig(ChatConfigKey.TeamInternal.GhostTextUseCompletionsFetchService, this.expService) + ? await this.fetcherService.fetchAndStreamCompletions2(completionParams, baseTelemetryData, finishedCb, cancellationToken) + : await this.fetcherService.fetchAndStreamCompletions(completionParams, baseTelemetryData, finishedCb, cancellationToken); + if (res.type === 'failed') { + return { + type: 'failed', + reason: res.reason, + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + }; + } + + if (res.type === 'canceled') { + ghostTextLogger.debug(this.logTarget, 'Cancelled after awaiting fetchCompletions'); + return { + type: 'canceled', + reason: res.reason, + telemetryData: mkCanceledResultTelemetry(baseTelemetryData), + }; + } + + return processChoices(requestStart, res.getProcessingTime(), res.choices); + } catch (err) { + // If we cancelled a network request, we don't want to log an error + if (isAbortError(err)) { + return { + type: 'canceled', + reason: 'network request aborted', + telemetryData: mkCanceledResultTelemetry(baseTelemetryData, { + cancelledNetworkRequest: true, + }), + }; + } else { + this.instantiationService.invokeFunction(acc => ghostTextLogger.exception(acc, err, `Error on ghost text request`)); + this.userErrorNotifier.notifyUser(err); + if (this.runtimeMode.shouldFailForDebugPurposes()) { + throw err; + } + // not including err in this result because it'll end up in standard telemetry + return { + type: 'failed', + reason: 'non-abort error on ghost text request', + telemetryData: mkBasicResultTelemetry(baseTelemetryData), + }; + } + } + } +} + +/** + * Post-proceses a completion choice based on the current request context and existing choices. + */ +function postProcessChoices( + newChoice: APIChoice, + currentChoices?: APIChoice[] +): APIChoice | undefined { + if (!currentChoices) { currentChoices = []; } + newChoice.completionText = newChoice.completionText.trimEnd(); + if (!newChoice.completionText) { return undefined; } + // Collect only unique displayTexts + if (currentChoices.findIndex(v => v.completionText.trim() === newChoice.completionText.trim()) !== -1) { + return undefined; + } + return newChoice; +} + +function makeGhostAPIChoice(choice: APIChoice, options: { forceSingleLine: boolean }): APIChoice { + const ghostChoice = { ...choice } as APIChoice; + if (options.forceSingleLine) { + const { completionText } = ghostChoice; + // Special case for when completion starts with a newline, don't count that as its own line + const initialLineBreak = completionText.match(/^\r?\n/); + if (initialLineBreak) { + ghostChoice.completionText = initialLineBreak[0] + completionText.split('\n')[1]; + } else { + ghostChoice.completionText = completionText.split('\n')[0]; + } + } + return ghostChoice; +} + +type GhostTextStrategy = { + blockMode: BlockMode; + requestMultiline: boolean; + finishedCb: FinishedCallback; + stop?: string[]; + maxTokens?: number; +}; + +function takeNLines(n: number): FinishedCallback { + return (text: string): number | undefined => { + // If the text is longer than n lines, return the offset. + // Checks for n+1 lines because of the leading newline. + const lines = text?.split('\n') ?? []; + if (lines.length > n + 1) { + return lines.slice(0, n + 1).join('\n').length; + } + }; +} + +async function getGhostTextStrategy( + accessor: ServicesAccessor, + completionState: CompletionState, + prefix: string, + prompt: PromptResponsePresent, + isCycling: boolean, + inlineSuggestion: boolean, + hasAcceptedCurrentCompletion: boolean, + preIssuedTelemetryData: TelemetryWithExp +): Promise<GhostTextStrategy> { + const instantiationService = accessor.get(IInstantiationService); + const featuresService = accessor.get(ICompletionsFeaturesService); + const blockModeConfig = accessor.get(ICompletionsBlockModeConfig); + const multilineAfterAcceptLines = featuresService.multilineAfterAcceptLines(preIssuedTelemetryData); + const blockMode = blockModeConfig.forLanguage(completionState.textDocument.detectedLanguageId, preIssuedTelemetryData); + switch (blockMode) { + case BlockMode.Server: + // Override the server-side trimming after accepting a completion + if (hasAcceptedCurrentCompletion) { + return { + blockMode: BlockMode.Parsing, + requestMultiline: true, + finishedCb: takeNLines(multilineAfterAcceptLines), + stop: ['\n\n'], + maxTokens: maxSinglelineTokens * multilineAfterAcceptLines, + }; + } + return { + blockMode: BlockMode.Server, + requestMultiline: true, + finishedCb: _ => undefined, + }; + case BlockMode.Parsing: + case BlockMode.ParsingAndServer: + case BlockMode.MoreMultiline: + default: { + // we shouldn't drop through to here, but in case we do, be explicit about the behaviour + let requestMultiline: MultilineDetermination; + try { + requestMultiline = await instantiationService.invokeFunction(shouldRequestMultiline, + blockMode, + completionState.textDocument, + completionState.position, + inlineSuggestion, + hasAcceptedCurrentCompletion, + prompt + ); + } catch (err) { + // Fallback to non-multiline + requestMultiline = { requestMultiline: false }; + } + if ( + !hasAcceptedCurrentCompletion && + requestMultiline.requestMultiline && + featuresService.singleLineUnlessAccepted(preIssuedTelemetryData) + ) { + requestMultiline.requestMultiline = false; + } + if (requestMultiline.requestMultiline) { + // Note that `trailingWs` contains *any* trailing whitespace from the prompt, but the prompt itself + // is only trimmed if the entire last line is whitespace. We have to account for that here when we + // check whether the block body is finished. + let adjustedPosition; + if (prompt.trailingWs.length > 0 && !prompt.prompt.prefix.endsWith(prompt.trailingWs)) { + // Prompt was adjusted, so adjust the position to match + adjustedPosition = LocationFactory.position( + completionState.position.line, + Math.max(completionState.position.character - prompt.trailingWs.length, 0) + ); + } else { + // Otherwise, just use the original position + adjustedPosition = completionState.position; + } + return { + blockMode: blockMode, + requestMultiline: true, + ...instantiationService.invokeFunction(buildFinishedCallback, + blockMode, + completionState.textDocument, + adjustedPosition, + requestMultiline.blockPosition, + prefix, + true, + prompt.prompt, + preIssuedTelemetryData + ), + }; + } + // Override single-line to multiline after accepting a completion + if (hasAcceptedCurrentCompletion) { + const result: GhostTextStrategy = { + blockMode: BlockMode.Parsing, + requestMultiline: true, + finishedCb: takeNLines(multilineAfterAcceptLines), + stop: ['\n\n'], + maxTokens: maxSinglelineTokens * multilineAfterAcceptLines, + }; + if (blockMode === BlockMode.MoreMultiline) { + result.blockMode = BlockMode.MoreMultiline; + } + return result; + } + // not multiline + return { + blockMode: blockMode, + requestMultiline: false, + ...instantiationService.invokeFunction(buildFinishedCallback, + blockMode, + completionState.textDocument, + completionState.position, + requestMultiline.blockPosition, + prefix, + false, + prompt.prompt, + preIssuedTelemetryData + ), + }; + } + } +} + +function buildFinishedCallback( + accessor: ServicesAccessor, + blockMode: BlockMode, + document: TextDocumentContents, + position: IPosition, + positionType: BlockPositionType | undefined, + prefix: string, + multiline: boolean, + prompt: Prompt, + telemetryData: TelemetryWithExp +): { finishedCb: FinishedCallback; maxTokens?: number } { + const featuresService = accessor.get(ICompletionsFeaturesService); + const instantiationService = accessor.get(IInstantiationService); + if (multiline && blockMode === BlockMode.MoreMultiline && BlockTrimmer.isSupported(document.detectedLanguageId)) { + const lookAhead = + positionType === BlockPositionType.EmptyBlock || positionType === BlockPositionType.BlockEnd + ? featuresService.longLookaheadSize(telemetryData) + : featuresService.shortLookaheadSize(telemetryData); + + const completionsCacheService = accessor.get(ICompletionsCacheService); + const finishedCb = instantiationService.createInstance(StreamedCompletionSplitter, + prefix, + document.detectedLanguageId, + false, + lookAhead, + (extraPrefix: string, item: APIChoice) => { + const cacheContext = { + prefix: prefix + extraPrefix, + prompt: { ...prompt, prefix: prompt.prefix + extraPrefix }, + }; + appendToCache(completionsCacheService, cacheContext, item); + } + ).getFinishedCallback(); + + return { + finishedCb, + maxTokens: featuresService.maxMultilineTokens(telemetryData), + }; + } + + return { finishedCb: multiline ? parsingBlockFinished(document, position) : _ => undefined }; +} + +export type GetGhostTextOptions = ExtractPromptOptions & { + /** Indicates if this is a cycling request. */ + isCycling: boolean; + /** Whether to stop the ghost text request after computing the prompt (used in the simulator) + */ + promptOnly: boolean; + /** + * Indicates if this is a speculative request generated assuming that the completion was accepted, + */ + isSpeculative: boolean; + /** + * Opportunity ID is a unique ID generated by the client relating to a + * single "opportunity" to provide some kind of suggestion to the user. + */ + opportunityId?: string; + /** + * An optional debounce time in milliseconds before requesting a completion. + * Overridable via config or exp variable: `copilotvscodedebouncethreshold`. + */ + debounceMs?: number; +}; + +const defaultOptions: GetGhostTextOptions = { + isCycling: false, + promptOnly: false, + isSpeculative: false, +}; + +function getRemainingDebounceMs(accessor: ServicesAccessor, opts: GetGhostTextOptions, telemetry: TelemetryWithExp): number { + const featuresService = accessor.get(ICompletionsFeaturesService); + const debounce = + getConfig<number | undefined>(accessor, ConfigKey.CompletionsDebounce) ?? + featuresService.completionsDebounce(telemetry) ?? + opts.debounceMs; + if (debounce === undefined) { return 0; } + const elapsed = now() - telemetry.issuedTime; + return Math.max(0, debounce - elapsed); +} + +function isCompletionRequestCancelled( + currentGhostText: ICompletionsCurrentGhostText, + requestId: string, + cancellationToken?: ICancellationToken +): boolean { + return cancellationToken?.isCancellationRequested || requestId !== currentGhostText.currentRequestId; +} + +async function getGhostTextWithoutAbortHandling( + accessor: ServicesAccessor, + completionState: CompletionState, + ourRequestId: string, + preIssuedTelemetryDataWithExp: TelemetryWithExp, + cancellationToken?: ICancellationToken, + options?: Partial<GetGhostTextOptions> +): Promise<GhostTextResultWithTelemetry<[CompletionResult[], ResultType]>> { + let start = preIssuedTelemetryDataWithExp.issuedTime; // Start before getting exp assignments + const performanceMetrics: [string, number][] = []; + /** Internal helper to record performance measurements. Mutates performanceMetrics and start. */ + function recordPerformance(name: string) { + const next = now(); + performanceMetrics.push([name, next - start]); + start = next; + } + recordPerformance('telemetry'); + const instantiationService = accessor.get(IInstantiationService); + const featuresService = accessor.get(ICompletionsFeaturesService); + const asyncCompletionManager = accessor.get(ICompletionsAsyncManagerService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const currentGhostText = accessor.get(ICompletionsCurrentGhostText); + const statusReporter = accessor.get(ICompletionsStatusReporter); + + if (isCompletionRequestCancelled(currentGhostText, ourRequestId, cancellationToken)) { + return { + type: 'abortedBeforeIssued', + reason: 'cancelled before extractPrompt', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + const inlineSuggestion = isInlineSuggestion(completionState.textDocument, completionState.position); + if (inlineSuggestion === undefined) { + ghostTextLogger.debug(logTarget, 'Breaking, invalid middle of the line'); + return { + type: 'abortedBeforeIssued', + reason: 'Invalid middle of the line', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + const engineInfo = instantiationService.invokeFunction(getEngineRequestInfo, preIssuedTelemetryDataWithExp); + const ghostTextOptions = { ...defaultOptions, ...options, tokenizer: engineInfo.tokenizer }; + const prompt = await instantiationService.invokeFunction(extractPrompt, + ourRequestId, + completionState, + preIssuedTelemetryDataWithExp, + undefined, + ghostTextOptions + ); + recordPerformance('prompt'); + if (prompt.type === 'copilotContentExclusion') { + ghostTextLogger.debug(logTarget, 'Copilot not available, due to content exclusion'); + return { + type: 'abortedBeforeIssued', + reason: 'Copilot not available due to content exclusion', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + if (prompt.type === 'contextTooShort') { + ghostTextLogger.debug(logTarget, 'Breaking, not enough context'); + return { + type: 'abortedBeforeIssued', + reason: 'Not enough context', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + if (prompt.type === 'promptError') { + ghostTextLogger.debug(logTarget, 'Error while building the prompt'); + return { + type: 'abortedBeforeIssued', + reason: 'Error while building the prompt', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + if (ghostTextOptions.promptOnly) { + return { type: 'promptOnly', reason: 'Breaking, promptOnly set to true', prompt: prompt }; + } + + if (prompt.type === 'promptCancelled') { + ghostTextLogger.debug(logTarget, 'Cancelled during extractPrompt'); + return { + type: 'abortedBeforeIssued', + reason: 'Cancelled during extractPrompt', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + if (prompt.type === 'promptTimeout') { + ghostTextLogger.debug(logTarget, 'Timeout during extractPrompt'); + return { + type: 'abortedBeforeIssued', + reason: 'Timeout', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + if (prompt.prompt.prefix.length === 0 && prompt.prompt.suffix.length === 0) { + ghostTextLogger.debug(logTarget, 'Error empty prompt'); + return { + type: 'abortedBeforeIssued', + reason: 'Empty prompt', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + + const debounce = instantiationService.invokeFunction(getRemainingDebounceMs, ghostTextOptions, preIssuedTelemetryDataWithExp); + if (debounce > 0) { + ghostTextLogger.debug(logTarget, `Debouncing ghost text request for ${debounce}ms`); + await delay(debounce); + if (isCompletionRequestCancelled(currentGhostText, ourRequestId, cancellationToken)) { + return { + type: 'abortedBeforeIssued', + reason: 'cancelled after debounce', + telemetryData: mkBasicResultTelemetry(preIssuedTelemetryDataWithExp), + }; + } + } + + return statusReporter.withProgress(async () => { + const [prefix] = trimLastLine( + completionState.textDocument.getText( + LocationFactory.range(LocationFactory.position(0, 0), completionState.position) + ) + ); + + const hasAcceptedCurrentCompletion = currentGhostText.hasAcceptedCurrentCompletion(prefix, prompt.prompt.suffix); + const originalPrompt = prompt.prompt; + const ghostTextStrategy = await instantiationService.invokeFunction(getGhostTextStrategy, + completionState, + prefix, + prompt, + ghostTextOptions.isCycling, + inlineSuggestion, + hasAcceptedCurrentCompletion, + preIssuedTelemetryDataWithExp + ); + recordPerformance('strategy'); + + let choices = instantiationService.invokeFunction(getLocalInlineSuggestion, prefix, originalPrompt, ghostTextStrategy.requestMultiline); + recordPerformance('cache'); + const repoInfo = instantiationService.invokeFunction(extractRepoInfoInBackground, completionState.textDocument.uri); + const requestContext: RequestContext = { + blockMode: ghostTextStrategy.blockMode, + languageId: completionState.textDocument.detectedLanguageId, + repoInfo: repoInfo, + engineModelId: engineInfo.modelId, + ourRequestId, + prefix, + prompt: prompt.prompt, + multiline: ghostTextStrategy.requestMultiline, + indentation: contextIndentation(completionState.textDocument, completionState.position), + isCycling: ghostTextOptions.isCycling, + headers: engineInfo.headers, + stop: ghostTextStrategy.stop, + maxTokens: ghostTextStrategy.maxTokens, + afterAccept: hasAcceptedCurrentCompletion, + }; + // Add headers to identify async completions and speculative requests + requestContext.headers = { + ...requestContext.headers, + 'X-Copilot-Async': 'true', + 'X-Copilot-Speculative': ghostTextOptions.isSpeculative ? 'true' : 'false', + }; + + // this will be used as basis for the choice telemetry data + const telemetryData = instantiationService.invokeFunction(telemetryIssued, + completionState.textDocument, + requestContext, + completionState.position, + prompt, + preIssuedTelemetryDataWithExp, + engineInfo, + ghostTextOptions + ); + + // Wait before requesting more completions if there is a candidate + // completion request in flight. Does not wait for cycling requests or + // if there is a cached completion. + if ( + choices === undefined && + !ghostTextOptions.isCycling && + asyncCompletionManager.shouldWaitForAsyncCompletions(prefix, prompt.prompt) + ) { + const choice = await asyncCompletionManager.getFirstMatchingRequestWithTimeout( + ourRequestId, + prefix, + prompt.prompt, + ghostTextOptions.isSpeculative, + telemetryData + ); + recordPerformance('asyncWait'); + if (choice) { + const forceSingleLine = !ghostTextStrategy.requestMultiline; + const trimmedChoice = makeGhostAPIChoice(choice[0], { forceSingleLine }); + choices = [[trimmedChoice], ResultType.Async]; + } + if (isCompletionRequestCancelled(currentGhostText, ourRequestId, cancellationToken)) { + ghostTextLogger.debug(logTarget, 'Cancelled before requesting a new completion'); + return { + type: 'abortedBeforeIssued', + reason: 'Cancelled after waiting for async completion', + telemetryData: mkBasicResultTelemetry(telemetryData), + }; + } + } + + const isMoreMultiline = + ghostTextStrategy.blockMode === BlockMode.MoreMultiline && + BlockTrimmer.isSupported(completionState.textDocument.detectedLanguageId); + if (choices !== undefined) { + // Post-process any cached choices before deciding whether to issue a network request + choices[0] = choices[0] + .map(c => + instantiationService.invokeFunction(postProcessChoiceInContext, + completionState.textDocument, + completionState.position, + c, + isMoreMultiline, + ghostTextLogger + ) + ) + .filter(c => c !== undefined); + } + + if (choices !== undefined && choices[0].length === 0) { + ghostTextLogger.debug(logTarget, `Found empty inline suggestions locally via ${resultTypeToString(choices[1])}`); + return { + type: 'empty', + reason: 'cached results empty after post-processing', + telemetryData: mkBasicResultTelemetry(telemetryData), + }; + } + if ( + choices !== undefined && + choices[0].length > 0 && + // If it's a cycling request, need to show multiple choices + (!ghostTextOptions.isCycling || choices[0].length > 1) + ) { + ghostTextLogger.debug(logTarget, `Found inline suggestions locally via ${resultTypeToString(choices[1])}`); + } else { + // No local choices, go to network + const completionsFromNetwork = instantiationService.createInstance(CompletionsFromNetwork); + if (ghostTextOptions.isCycling) { + const networkChoices = await completionsFromNetwork.getAllCompletionsFromNetwork( + requestContext, + telemetryData, + cancellationToken, + ghostTextStrategy.finishedCb + ); + + // TODO: if we already had some choices cached from the initial non-cycling request, + // and then the cycling request returns no results for some reason, we need to still + // return the original choices to the editor to avoid the ghost text disappearing completely. + // However this should be telemetrised according to the result of the cycling request itself, + // i.e. failure/empty (or maybe canceled). + // + // Right now this is awkward to orchestrate in the code and we don't handle it, incorrectly + // returning `ghostText.produced` instead. Cycling is a manual action and hence uncommon, + // so this shouldn't cause much inaccuracy, but we still should fix this. + if (networkChoices.type === 'success') { + const resultChoices = choices?.[0] ?? []; + networkChoices.value[0].forEach(c => { + // Collect only unique displayTexts + if (resultChoices.findIndex(v => v.completionText.trim() === c.completionText.trim()) !== -1) { + return; + } + resultChoices.push(c); + }); + choices = [resultChoices, ResultType.Cycling]; + } else { + if (choices === undefined) { + return networkChoices; + } + } + } else { + // Wrap an observer around the finished callback to update the + // async manager as the request streams in. + const finishedCb: FinishedCallback = (text, delta) => { + asyncCompletionManager.updateCompletion(ourRequestId, text); + return ghostTextStrategy.finishedCb(text, delta); + }; + + const asyncCancellationTokenSource = new CancellationTokenSource(); + const requestPromise = completionsFromNetwork.getCompletionsFromNetwork( + requestContext, + telemetryData, + asyncCancellationTokenSource.token, + finishedCb + ); + void asyncCompletionManager.queueCompletionRequest( + ourRequestId, + prefix, + prompt.prompt, + asyncCancellationTokenSource, + requestPromise + ); + const c = await asyncCompletionManager.getFirstMatchingRequest(ourRequestId, prefix, prompt.prompt, ghostTextOptions.isSpeculative); + if (c === undefined) { + return { + type: 'empty', + reason: 'received no results from async completions', + telemetryData: mkBasicResultTelemetry(telemetryData), + }; + } + choices = [[c[0]], ResultType.Async]; + } + recordPerformance('network'); + } + if (choices === undefined) { + return { + type: 'failed', + reason: 'internal error: choices should be defined after network call', + telemetryData: mkBasicResultTelemetry(telemetryData), + }; + } + const [choicesArray, resultType] = choices; + + const postProcessedChoicesArray = choicesArray + .map(c => + instantiationService.invokeFunction(postProcessChoiceInContext, + completionState.textDocument, + completionState.position, + c, + isMoreMultiline, + ghostTextLogger + ) + ) + .filter(c => c !== undefined); + + // Delay response if needed. Note, this must come before the + // telemetryWithAddData call since the time_to_produce_ms is computed + // there + const completionsDelay = + instantiationService.invokeFunction(getConfig<number>, ConfigKey.CompletionsDelay) ?? + featuresService.completionsDelay(preIssuedTelemetryDataWithExp); + const elapsed = now() - preIssuedTelemetryDataWithExp.issuedTime; + const remainingDelay = Math.max(completionsDelay - elapsed, 0); + if (resultType !== ResultType.TypingAsSuggested && !ghostTextOptions.isCycling && remainingDelay > 0) { + ghostTextLogger.debug(logTarget, `Waiting ${remainingDelay}ms before returning completion`); + await delay(remainingDelay); + if (isCompletionRequestCancelled(currentGhostText, ourRequestId, cancellationToken)) { + ghostTextLogger.debug(logTarget, 'Cancelled after completions delay'); + return { + type: 'canceled', + reason: 'after completions delay', + telemetryData: mkCanceledResultTelemetry(telemetryData), + }; + } + } + + const results: CompletionResult[] = []; + for (const choice of postProcessedChoicesArray) { + // Do this to get a new object for each choice + const choiceTelemetryData = telemetryWithAddData( + completionState.textDocument, + requestContext, + choice, + telemetryData + ); + + const suffixCoverage = inlineSuggestion + ? checkSuffix(completionState.textDocument, completionState.position, choice) + : 0; + + // We want to use `newTrailingWs` as the trailing whitespace + const ghostCompletion = adjustLeadingWhitespace( + choice.choiceIndex, + choice.completionText, + prompt.trailingWs + ); + const res: CompletionResult = { + completion: ghostCompletion, + telemetry: choiceTelemetryData, + isMiddleOfTheLine: inlineSuggestion, + suffixCoverage, + copilotAnnotations: choice.copilotAnnotations, + clientCompletionId: choice.clientCompletionId, + }; + results.push(res); + } + + // Lift clientCompletionId out of the result in order to include it in the telemetry payload computed by mkBasicResultTelemetry. + telemetryData.properties.clientCompletionId = results[0]?.clientCompletionId; + // If reading from the cache or async, capture the look back offset used + telemetryData.measurements.foundOffset = results?.[0]?.telemetry?.measurements?.foundOffset ?? -1; + ghostTextLogger.debug( + logTarget, + `Produced ${results.length} results from ${resultTypeToString(resultType)} at ${telemetryData.measurements.foundOffset} offset` + ); + + if (isCompletionRequestCancelled(currentGhostText, ourRequestId, cancellationToken)) { + return { + type: 'canceled', + reason: 'after post processing completions', + telemetryData: mkCanceledResultTelemetry(telemetryData), + }; + } + + if (!ghostTextOptions.isSpeculative) { + // Update the current ghost text with the new response before returning for the "typing as suggested" UX + currentGhostText.setGhostText(prefix, prompt.prompt.suffix, postProcessedChoicesArray, resultType); + } + + recordPerformance('complete'); + + return { + type: 'success', + value: [results, resultType], + telemetryData: mkBasicResultTelemetry(telemetryData), + telemetryBlob: telemetryData, + resultType, + performanceMetrics, + }; + }); +} + +export async function getGhostText( + accessor: ServicesAccessor, + completionState: CompletionState, + token: ICancellationToken | undefined, + options: Partial<GetGhostTextOptions> +): Promise<GhostTextResultWithTelemetry<[CompletionResult[], ResultType]>> { + const id = generateUuid(); + const instantiationService = accessor.get(IInstantiationService); + const telemetryService = accessor.get(ITelemetryService); + const notifierService = accessor.get(ICompletionsNotifierService); + const contextProviderBridge = accessor.get(ICompletionsContextProviderBridgeService); + const currentGhostText = accessor.get(ICompletionsCurrentGhostText); + const contextproviderStatistics = accessor.get(ICompletionsContextProviderService); + currentGhostText.currentRequestId = id; + const telemetryData = await createTelemetryWithExp(accessor, completionState.textDocument, id, options); + // A CLS consumer has an LSP bug where it erroneously makes method requests before `initialize` has returned, which + // means we can't use `initialize` to actually initialize anything expensive. This the primary user of the + // tokenizer, so settle for initializing here instead. We don't use waitForTokenizers() because in the event of a + // tokenizer load failure, that would spam handleException() on every request. + await initializeTokenizers.catch(() => { }); + try { + contextProviderBridge.schedule( + completionState, + id, + options?.opportunityId ?? '', + telemetryData, + token, + options + ); + notifierService.notifyRequest(completionState, id, telemetryData, token, options); + const result = await instantiationService.invokeFunction(getGhostTextWithoutAbortHandling, completionState, id, telemetryData, token, options); + const statistics = contextproviderStatistics.getStatisticsForCompletion(id); + const opportunityId = options?.opportunityId ?? 'unknown'; + for (const [providerId, statistic] of statistics.getAllUsageStatistics()) { + /* __GDPR__ + "context-provider.completion-stats" : { + "owner": "dirkb", + "comment": "Telemetry for copilot inline completion context", + "requestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The request correlation id" }, + "opportunityId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The opportunity id" }, + "providerId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The context provider id" }, + "resolution": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The resolution of the context" }, + "usage": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "How the context was used" }, + "usageDetails": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Additional details about the usage as a JSON string" } + } + */ + telemetryService.sendMSFTTelemetryEvent( + 'context-provider.completion-stats', + { + requestId: id, + opportunityId, + providerId, + resolution: statistic.resolution, + usage: statistic.usage, + usageDetails: JSON.stringify(statistic.usageDetails), + }, + { + } + ); + } + return result; + } catch (e) { + // The cancellation token may be called after the request is done but while we still process data. + // The underlying implementation catches abort errors for specific scenarios but we still have uncovered paths. + // To avoid returning an error to the editor, this acts as an fault barrier here. + if (isAbortError(e)) { + return { + type: 'canceled', + reason: 'aborted at unknown location', + telemetryData: mkCanceledResultTelemetry(telemetryData, { + cancelledNetworkRequest: true, + }), + }; + } + throw e; + } +} + +/** + * Attempt to get InlineSuggestion locally, in one of two ways: + * 1. If the user is typing the letters already displayed as inline suggestion. + * 2. If we have a previously cached inline suggestion for this prompt and requestMultiline. + */ +function getLocalInlineSuggestion( + accessor: ServicesAccessor, + prefix: string, + prompt: Prompt, + requestMultiline: boolean +): [APIChoice[], ResultType] | undefined { + const currentGhostText = accessor.get(ICompletionsCurrentGhostText); + const choicesTyping = currentGhostText.getCompletionsForUserTyping(prefix, prompt.suffix); + const choicesCache = getCompletionsFromCache(accessor, prefix, prompt.suffix, requestMultiline); + + if (choicesTyping && choicesTyping.length > 0) { + // Append cached choices to choicesTyping, if any. Ensure typing choices + // are first so that the shown completion doesn't disappear. + // Filter duplicates by completionText + const choicesCacheDeduped = (choicesCache ?? []).filter( + c => !choicesTyping.some(t => t.completionText === c.completionText) + ); + return [choicesTyping.concat(choicesCacheDeduped), ResultType.TypingAsSuggested]; + } + + if (choicesCache && choicesCache.length > 0) { + return [choicesCache, ResultType.Cache]; + } +} + +/** Info for caching completions. */ +interface CacheContext { + /** The text content up to the cursor. */ + prefix: string; + /** The prompt to send to the model. */ + prompt: Prompt; + /** + * If true, add an extra newline at the end of the prefix of the prompt. This is used to get a completion for the next line. + * Unset if the feature is disabled. + */ + requestForNextLine?: boolean; +} + +/** Info for requesting and caching completions. */ +interface RequestContext { + /** How block trimming should be done. */ + blockMode: BlockMode; + /** The language of the file. */ + languageId: string; + /** Information about the repository the file is in, if available. */ + repoInfo: MaybeRepoInfo; + /** The engine used for the request. */ + engineModelId: string; + /** A request id we choose in the hope that the model will use it in responses */ + ourRequestId: string; + /** The text content up to the cursor. */ + prefix: string; + /** The prompt to send to the model. */ + prompt: Prompt; + /** Whether this request should be able to generate multiple lines. */ + multiline: boolean; + /** Indentation (tabs or spaces) on/before and after the cursor. */ + indentation: ContextIndentation; + /** Follow up request happening when user requested cycling */ + isCycling: boolean; + /** Additional request headers */ + headers: CompletionHeaders; + /** Optional override for the default stop sequences for this request. */ + stop?: string[]; + /** Optional override for max tokens to return */ + maxTokens?: number; + /** Whether the current request is following an accepted completion. */ + afterAccept: boolean; +} + +/** Checks if the position is valid inline suggestion position. Returns `undefined` if it's position where ghost text shouldn't be displayed */ +function isInlineSuggestion(document: TextDocumentContents, position: IPosition) { + //Checks if we're in the position for the middle of the line suggestion + const isMiddleOfLine = isMiddleOfTheLine(position, document); + const isValidMiddleOfLine = isValidMiddleOfTheLinePosition(position, document); + + if (isMiddleOfLine && !isValidMiddleOfLine) { + return; + } + + const isInlineSuggestion = isMiddleOfLine && isValidMiddleOfLine; + return isInlineSuggestion; +} + +/** Checks if position is NOT at the end of the line */ +function isMiddleOfTheLine(selectionPosition: IPosition, doc: TextDocumentContents): boolean { + // must be end of line or trailing whitespace + const line = doc.lineAt(selectionPosition); + if (line.text.substr(selectionPosition.character).trim().length !== 0) { + return true; + } + + return false; +} + +/** Checks if position is valid for the middle of the line suggestion */ +function isValidMiddleOfTheLinePosition(selectionPosition: IPosition, doc: TextDocumentContents): boolean { + const line = doc.lineAt(selectionPosition); + const endOfLine = line.text.substr(selectionPosition.character).trim(); + return /^\s*[)>}\]"'`]*\s*[:{;,]?\s*$/.test(endOfLine); +} + +/** Checks if position is the beginning of an empty line (including indentation) */ +function isNewLine(selectionPosition: IPosition, doc: TextDocumentContents): boolean { + const line = doc.lineAt(selectionPosition); + const lineTrimmed = line.text.trim(); + return lineTrimmed.length === 0; +} + +// This enables tests to control multi line behavior +export class ForceMultiLine { + static readonly default = new ForceMultiLine(); + + constructor(readonly requestMultilineOverride = false) { } +} + +type MultilineDetermination = { + requestMultiline: boolean; + blockPosition?: BlockPositionType; +}; + +async function shouldRequestMultiline( + accessor: ServicesAccessor, + blockMode: BlockMode, + document: TextDocumentContents, + position: IPosition, + inlineSuggestion: boolean, + afterAccept: boolean, + prompt: PromptResponsePresent +): Promise<MultilineDetermination> { + + // Parsing long files for multiline completions is slow, so we only do + // it for files with less than 8000 lines + if (document.lineCount >= 8000) { + telemetry( + accessor, + 'ghostText.longFileMultilineSkip', + TelemetryData.createAndMarkAsIssued({ + languageId: document.detectedLanguageId, + lineCount: String(document.lineCount), + currentLine: String(position.line), + }) + ); + } else { + if (blockMode === BlockMode.MoreMultiline && BlockTrimmer.isSupported(document.detectedLanguageId)) { + if (!afterAccept) { + return { requestMultiline: false }; + } + const blockPosition = await getBlockPositionType(document, position); + return { requestMultiline: true, blockPosition }; + } + + const targetLanguagesNewLine = ['typescript', 'typescriptreact']; + if (targetLanguagesNewLine.includes(document.detectedLanguageId)) { + const newLine = isNewLine(position, document); + if (newLine) { + return { requestMultiline: true }; + } + } + let requestMultiline = false; + if (!inlineSuggestion && isSupportedLanguageId(document.detectedLanguageId)) { + // Can only check block-level nodes of languages we support + requestMultiline = await isEmptyBlockStartUtil(document, position); + } else if (inlineSuggestion && isSupportedLanguageId(document.detectedLanguageId)) { + //If we are inline, check if we would suggest multiline for current position or if we would suggest a multiline completion if we were at the end of the line + requestMultiline = + (await isEmptyBlockStartUtil(document, position)) || + (await isEmptyBlockStartUtil(document, document.lineAt(position).range.end)); + } + // If requestMultiline is false, for specific languages check multiline score + if (!requestMultiline) { + const requestMultiModelThreshold = 0.5; + const targetLanguagesModel = ['javascript', 'javascriptreact', 'python']; + if (targetLanguagesModel.includes(document.detectedLanguageId)) { + // Call multiline model if not multiline and EXP flag is set. + const multiModelScore = requestMultilineScore(prompt.prompt, document.detectedLanguageId); + requestMultiline = multiModelScore > requestMultiModelThreshold; + } + } + return { requestMultiline }; + } + return { requestMultiline: false }; +} + +/** Appends completions to existing entry in cache or creates new entry. */ +function appendToCache(competionsCacheService: ICompletionsCacheService, requestContext: CacheContext, choice: APIChoice) { + competionsCacheService.append(requestContext.prefix, requestContext.prompt.suffix, choice); +} + +function adjustLeadingWhitespace(index: number, text: string, ws: string): GhostCompletion { + if (ws.length > 0) { + if (text.startsWith(ws)) { + // Remove common prefix so that it can display in the correct position + return { + completionIndex: index, + completionText: text, + displayText: text.substring(ws.length), + displayNeedsWsOffset: false, + }; + } else { + // The idea here is that we do want the display to be as close to the final position as possible + const textLeftWs = text.substring(0, text.length - text.trimStart().length); + if (ws.startsWith(textLeftWs)) { + // NOTE: It's possible that `ws` is a bit too over-indented. Example: + // def foo(n): + // if n > 0: + // print(f"n is positive: {n}") + // [cursor is here after new line] + // + // completion: " else:" + return { + completionIndex: index, + completionText: text, + displayText: text.trimStart(), + displayNeedsWsOffset: true, + }; + } else { + // We don't know any better so just send `text` back + return { completionIndex: index, completionText: text, displayText: text, displayNeedsWsOffset: false }; + } + } + } else { + // If we do not know leading whitespace or if it is an empty string, just return input text + return { completionIndex: index, completionText: text, displayText: text, displayNeedsWsOffset: false }; + } +} + +/** + * Returns all completions from the cache for given document prefix. Walks back + * from the current prefix to search for completions with a prefix that + * partially matches the current prefix and completion text that matches the + * remaining current prefix. + */ +function getCompletionsFromCache( + accessor: ServicesAccessor, + prefix: string, + suffix: string, + multiline: boolean +): APIChoice[] | undefined { + const logTarget = accessor.get(ICompletionsLogTargetService); + const choices = accessor.get(ICompletionsCacheService).findAll(prefix, suffix); + if (choices.length === 0) { + ghostTextLogger.debug(logTarget, `Found no completions in cache`); + return []; + } + ghostTextLogger.debug(logTarget, `Found ${choices.length} completions in cache`); + return choices.map(choice => makeGhostAPIChoice(choice, { forceSingleLine: !multiline })); +} + +/** Create a TelemetryWithExp instance for a ghost text request. */ +async function createTelemetryWithExp( + accessor: ServicesAccessor, + document: TextDocumentContents, + headerRequestId: string, + options?: Partial<GetGhostTextOptions> +): Promise<TelemetryWithExp> { + const featuresService = accessor.get(ICompletionsFeaturesService); + const properties: TelemetryProperties = { headerRequestId }; + if (options?.opportunityId) { properties.opportunityId = options.opportunityId; } + if (options?.selectedCompletionInfo?.text) { properties.completionsActive = 'true'; } + if (options?.isSpeculative) { properties.reason = 'speculative'; } + const telemetryData = TelemetryData.createAndMarkAsIssued(properties); + const telemetryWithExp = await featuresService.updateExPValuesAndAssignments( + { uri: document.uri, languageId: document.detectedLanguageId }, + telemetryData + ); + return telemetryWithExp; +} + +/** Return a copy of the choice's telemetry data with extra information added */ +function telemetryWithAddData( + document: TextDocumentContents, + requestContext: RequestContext, + choice: APIChoice, + issuedTelemetryData: TelemetryWithExp +): TelemetryWithExp { + const requestId = choice.requestId; + const properties: { [key: string]: string } = { + choiceIndex: choice.choiceIndex.toString(), + clientCompletionId: choice.clientCompletionId, + }; + if (choice.generatedChoiceIndex !== undefined) { + properties.originalChoiceIndex = properties.choiceIndex; + properties.choiceIndex = (10_000 * (choice.generatedChoiceIndex + 1) + choice.choiceIndex).toString(); + } + const measurements: { [key: string]: number } = { + compCharLen: choice.completionText.length, + numLines: choice.completionText.trim().split('\n').length, + }; + // Add assessments + if (choice.meanLogProb) { + measurements.meanLogProb = choice.meanLogProb; + } + if (choice.meanAlternativeLogProb) { + measurements.meanAlternativeLogProb = choice.meanAlternativeLogProb; + } + + const extendedTelemetry = choice.telemetryData.extendedBy(properties, measurements); + extendedTelemetry.issuedTime = issuedTelemetryData.issuedTime; + extendedTelemetry.measurements.timeToProduceMs = performance.now() - issuedTelemetryData.issuedTime; + addDocumentTelemetry(extendedTelemetry, document); + extendedTelemetry.extendWithRequestId(requestId); + return extendedTelemetry; +} + +/** Create new telemetry data based on baseTelemetryData and send `ghostText.issued` event */ +function telemetryIssued( + accessor: ServicesAccessor, + document: TextDocumentContents, + requestContext: RequestContext, + position: IPosition, + prompt: PromptResponsePresent, + baseTelemetryData: TelemetryWithExp, + requestInfo: EngineRequestInfo, + ghostTextOptions: GetGhostTextOptions +): TelemetryWithExp { + // base ghostText telemetry data + const properties: { [key: string]: string } = { + languageId: document.detectedLanguageId, + }; + properties.afterAccept = requestContext.afterAccept.toString(); + properties.isSpeculative = ghostTextOptions.isSpeculative.toString(); + const telemetryData = baseTelemetryData.extendedBy(properties); + addDocumentTelemetry(telemetryData, document); + + // Add repository information + const repoInfo = requestContext.repoInfo; + telemetryData.properties.gitRepoInformation = + repoInfo === undefined ? 'unavailable' : repoInfo === ComputationStatus.PENDING ? 'pending' : 'available'; + if (repoInfo !== undefined && repoInfo !== ComputationStatus.PENDING) { + telemetryData.properties.gitRepoUrl = repoInfo.url; + telemetryData.properties.gitRepoHost = repoInfo.hostname; + if (repoInfo.repoId?.type === 'github') { + telemetryData.properties.gitRepoOwner = repoInfo.repoId.org; + telemetryData.properties.gitRepoName = repoInfo.repoId.repo; + } else if (repoInfo.repoId?.type === 'ado') { + telemetryData.properties.gitRepoOwner = repoInfo.repoId.project; + telemetryData.properties.gitRepoName = repoInfo.repoId.repo; + } else { + // TODO: We don't have generic owner and repo for other providers + } + telemetryData.properties.gitRepoPath = repoInfo.pathname; + } + + telemetryData.properties.engineName = requestInfo.modelId; + telemetryData.properties.engineChoiceSource = requestInfo.engineChoiceSource; + + // Add requestMultiline information + telemetryData.properties.isMultiline = JSON.stringify(requestContext.multiline); + telemetryData.properties.isCycling = JSON.stringify(requestContext.isCycling); + + // calculated values for the issued event + const currentLine = document.lineAt(position.line); + const lineBeforeCursor = document.getText(LocationFactory.range(currentLine.range.start, position)); + const restOfLine = document.getText(LocationFactory.range(position, currentLine.range.end)); + + const typeFileHashCode = Array.from(prompt.neighborSource.entries()).map(typeFiles => [ + typeFiles[0], + typeFiles[1].map(f => createSha256Hash(f).toString()), // file name is sensitive. We just keep SHA256 of the file name. + ]); + + // Properties that we only want to include in the issued event + const extendedProperties: TelemetryProperties = { + beforeCursorWhitespace: JSON.stringify(lineBeforeCursor.trim() === ''), + afterCursorWhitespace: JSON.stringify(restOfLine.trim() === ''), + neighborSource: JSON.stringify(typeFileHashCode), + blockMode: requestContext.blockMode, + }; + const extendedMeasurements: TelemetryMeasurements = { + ...telemetrizePromptLength(prompt.prompt), + promptEndPos: document.offsetAt(position), + promptComputeTimeMs: prompt.computeTimeMs, + }; + if (prompt.metadata) { + extendedProperties.promptMetadata = JSON.stringify(prompt.metadata); + } + if (prompt.contextProvidersTelemetry) { + extendedProperties.contextProviders = JSON.stringify(prompt.contextProvidersTelemetry); + } + const telemetryDataToSend = telemetryData.extendedBy(extendedProperties, extendedMeasurements); + + // telemetrize the issued event + telemetry(accessor, 'ghostText.issued', telemetryDataToSend); + + return telemetryData; +} + +function addDocumentTelemetry(telemetry: TelemetryWithExp, document: TextDocumentContents): void { + telemetry.measurements.documentLength = document.getText().length; + telemetry.measurements.documentLineCount = document.lineCount; +} + +function telemetryPerformance( + accessor: ServicesAccessor, + performanceKind: string, + choice: APIChoice, + requestStart: number, + processingTimeMs: number +) { + const requestTimeMs = Date.now() - requestStart; + const deltaMs = requestTimeMs - processingTimeMs; + + const telemetryData = choice.telemetryData.extendedBy( + {}, + { + completionCharLen: choice.completionText.length, + requestTimeMs: requestTimeMs, + processingTimeMs: processingTimeMs, + deltaMs: deltaMs, + // Choice properties + meanLogProb: choice.meanLogProb || NaN, + meanAlternativeLogProb: choice.meanAlternativeLogProb || NaN, + } + ); + telemetryData.extendWithRequestId(choice.requestId); + telemetry(accessor, `ghostText.${performanceKind}`, telemetryData); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/last.ts b/completions-sample-code/vscode-node/lib/src/ghostText/last.ts new file mode 100644 index 0000000..357996c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/last.ts @@ -0,0 +1,269 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { postInsertionTasks, postRejectionTasks } from '../postInsertion'; +import { countLines, SuggestionStatus } from '../suggestions/partialSuggestions'; +import { TelemetryWithExp } from '../telemetry'; +import { IPosition, TextDocumentContents, TextDocumentIdentifier } from '../textDocument'; +import { CopilotCompletion } from './copilotCompletion'; +import { ResultType } from './ghostText'; +import { GHOST_TEXT_CATEGORY, telemetryShown } from './telemetry'; + +const ghostTextLogger = new Logger('ghostText'); + +export const ICompletionsLastGhostText = createServiceIdentifier<ICompletionsLastGhostText>('ICompletionsLastGhostText'); +export interface ICompletionsLastGhostText { + readonly _serviceBrand: undefined; + + position: IPosition | undefined; + uri: string | undefined; + shownCompletions: CopilotCompletion[]; + index: number | undefined; + totalLength: number | undefined; + partiallyAcceptedLength: number | undefined; + linesLeft: number | undefined; + linesAccepted: number; + lastLineAcceptedLength: number | undefined; + + resetState(): void; + setState(document: TextDocumentIdentifier, position: IPosition): void; + resetPartialAcceptanceState(): void; +} + +export class LastGhostText implements ICompletionsLastGhostText { + declare _serviceBrand: undefined; + + #position: IPosition | undefined; + #uri: string | undefined; + #shownCompletions: CopilotCompletion[] = []; + index: number | undefined; + totalLength: number | undefined; + partiallyAcceptedLength: number | undefined; + linesLeft: number | undefined; // Lines left to accept in the current completion, used for partial acceptance + linesAccepted: number = 0; // Number of lines accepted in the current completion, used for partial acceptance + lastLineAcceptedLength: number | undefined; // Length of the last accepted line, used for partial acceptance + + get position() { + return this.#position; + } + + get shownCompletions() { + return this.#shownCompletions || []; + } + + get uri() { + return this.#uri; + } + + resetState() { + this.#uri = undefined; + this.#position = undefined; + this.#shownCompletions = []; + this.resetPartialAcceptanceState(); + } + + setState({ uri }: TextDocumentIdentifier, position: IPosition) { + this.#uri = uri; + this.#position = position; + this.#shownCompletions = []; + } + + resetPartialAcceptanceState() { + this.partiallyAcceptedLength = 0; + this.totalLength = undefined; + this.linesLeft = undefined; + this.linesAccepted = 0; + } +} + +function computeRejectedCompletions< + T extends { completionText: string; completionTelemetryData: TelemetryWithExp; offset: number }, +>(last: ICompletionsLastGhostText): T[] { + const rejectedCompletions: T[] = []; + last.shownCompletions.forEach(c => { + if (c.displayText && c.telemetry) { + let completionText; + let completionTelemetryData; + + if (last.partiallyAcceptedLength) { + // suggestion got partially accepted already but rejecting the remainder + completionText = c.displayText.substring(last.partiallyAcceptedLength - 1); + completionTelemetryData = c.telemetry.extendedBy( + { + compType: 'partial', + }, + { + compCharLen: completionText.length, + } + ); + } else { + completionText = c.displayText; + completionTelemetryData = c.telemetry; + } + const rejection = { completionText, completionTelemetryData, offset: c.offset }; + rejectedCompletions.push(rejection as T); + } + }); + return rejectedCompletions; +} + +export function rejectLastShown(accessor: ServicesAccessor, offset?: number) { + const last = accessor.get(ICompletionsLastGhostText); + if (!last.position || !last.uri) { return; } + //The position has changed and we're not in typing-as-suggested flow + // so previously shown completions can be reported as rejected + const rejectedCompletions = computeRejectedCompletions(last); + if (rejectedCompletions.length > 0) { + postRejectionTasks(accessor, 'ghostText', offset ?? rejectedCompletions[0].offset, last.uri, rejectedCompletions); + } + last.resetState(); + last.resetPartialAcceptanceState(); +} + +export function setLastShown( + accessor: ServicesAccessor, + document: TextDocumentContents, + position: IPosition, + resultType: ResultType +) { + const last = accessor.get(ICompletionsLastGhostText); + if ( + last.position && + last.uri && + !( + last.position.line === position.line && + last.position.character === position.character && + last.uri.toString() === document.uri.toString() + ) && + resultType !== ResultType.TypingAsSuggested // results for partial acceptance count as TypingAsSuggested + ) { + rejectLastShown(accessor, document.offsetAt(last.position)); + } + last.setState(document, position); + return last.index; +} + +export function handleGhostTextShown(accessor: ServicesAccessor, cmp: CopilotCompletion) { + const logTarget = accessor.get(ICompletionsLogTargetService); + const last = accessor.get(ICompletionsLastGhostText); + last.index = cmp.index; + if (!last.shownCompletions.find(c => c.index === cmp.index)) { + // Only update if .position is still at the position of the completion + if ( + cmp.uri === last.uri && + last.position?.line === cmp.position.line && + last.position?.character === cmp.position.character + ) { + last.shownCompletions.push(cmp); + } + // Show telemetry only if it was not shown before (i.e. don't sent repeated telemetry in cycling case when user cycled through every suggestions or goes back and forth) + if (cmp.displayText) { + const fromCache = !(cmp.resultType === ResultType.Network); + ghostTextLogger.debug( + logTarget, + `[${cmp.telemetry.properties.headerRequestId}] shown choiceIndex: ${cmp.telemetry.properties.choiceIndex}, fromCache ${fromCache}` + ); + cmp.telemetry.measurements.compCharLen = cmp.displayText.length; + telemetryShown(accessor, cmp); + } + } +} + +/** + * Handles partial acceptance for VS Code clients using line-based strategy. + * VS Code tracks acceptance by lines and resets the accepted length per line. + */ +function handleLineAcceptance(accessor: ServicesAccessor, cmp: CopilotCompletion, acceptedLength: number) { + const last = accessor.get(ICompletionsLastGhostText); + + // If this is the first acceptance, we need to initialize the linesLeft + if (last.linesLeft === undefined) { + last.linesAccepted = countLines(cmp.insertText.substring(0, acceptedLength)); + last.linesLeft = countLines(cmp.displayText); + } + + const linesLeft = countLines(cmp.displayText); + + if (last.linesLeft > linesLeft) { + // If the number of lines left has decreased, we need to update the accepted lines count + // and reset the last line accepted length + last.linesAccepted += last.linesLeft - linesLeft; + last.lastLineAcceptedLength = last.partiallyAcceptedLength; + last.linesLeft = linesLeft; + } + + last.partiallyAcceptedLength = (last.lastLineAcceptedLength || 0) + acceptedLength; +} + +/** + * Handles full acceptance of ghost text completions. + * This method is primarily used by VS Code for explicit full acceptances. + */ +export function handleGhostTextPostInsert( + accessor: ServicesAccessor, + cmp: CopilotCompletion, +) { + const last = accessor.get(ICompletionsLastGhostText); + + let suggestionStatus: SuggestionStatus; + + if (last.partiallyAcceptedLength) { + suggestionStatus = { + compType: 'full', + acceptedLength: (last.partiallyAcceptedLength || 0) + cmp.displayText.length, + acceptedLines: last.linesAccepted + (last.linesLeft ?? 0), + }; + } else { + suggestionStatus = { + compType: 'full', + acceptedLength: cmp.displayText.length, + acceptedLines: countLines(cmp.displayText), + }; + } + + //If any completion was accepted, clear the list of shown completions + //that would be passed to rejected telemetry + last.resetState(); + + return postInsertionTasks( + accessor, + GHOST_TEXT_CATEGORY, + cmp.displayText, + cmp.offset, + cmp.uri, + cmp.telemetry, + suggestionStatus, + cmp.copilotAnnotations + ); +} + +export function handlePartialGhostTextPostInsert( + accessor: ServicesAccessor, + cmp: CopilotCompletion, + acceptedLength: number, +) { + const last = accessor.get(ICompletionsLastGhostText); + + handleLineAcceptance(accessor, cmp, acceptedLength); + + const suggestionStatus: SuggestionStatus = { + compType: 'partial', + acceptedLength: last.partiallyAcceptedLength || 0, + acceptedLines: last.linesAccepted, + }; + + return postInsertionTasks( + accessor, + GHOST_TEXT_CATEGORY, + cmp.displayText, + cmp.offset, + cmp.uri, + cmp.telemetry, + suggestionStatus, + cmp.copilotAnnotations + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/multilineModel.ts b/completions-sample-code/vscode-node/lib/src/ghostText/multilineModel.ts new file mode 100644 index 0000000..ce7c2eb --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/multilineModel.ts @@ -0,0 +1,188 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Prompt } from '../prompt/prompt'; +import { contextualFilterCharacterMap } from './contextualFilterConstants'; +import { multilineModelPredict } from './multilineModelWeights'; + +// This comment map is based on the preprocessing used in training data. +// It should not be changed without a corresponding change in model training. +const commentMap: { [key: string]: string[] } = { + javascript: ['//'], + typescript: ['//'], + typescriptreact: ['//'], + javascriptreact: ['//'], + vue: ['//', '-->'], + php: ['//', '#'], + dart: ['//'], + go: ['//'], + cpp: ['//'], + scss: ['//'], + csharp: ['//'], + java: ['//'], + c: ['//'], + rust: ['//'], + python: ['#'], + markdown: ['#', '-->'], + css: ['*/'], +}; + +// This language map is based on the preprocessing used in training data. +// It should not be changed without a corresponding change in model training. +const languageMap: { [key: string]: number } = { + javascript: 1, + javascriptreact: 2, + typescript: 3, + typescriptreact: 4, + python: 5, + go: 6, + ruby: 7, +}; + +export function hasComment(text: string, lineNumber: number, language: string, ignoreEmptyLines = true): boolean { + let lines = text.split('\n'); + if (ignoreEmptyLines) { + lines = lines.filter(line => line.trim().length > 0); + } + if (Math.abs(lineNumber) > lines.length || lineNumber >= lines.length) { + return false; + } + if (lineNumber < 0) { + lineNumber = lines.length + lineNumber; + } + const line = lines[lineNumber]; + const commentChars = commentMap[language] ?? []; + return commentChars.some(commentChar => line.includes(commentChar)); +} + +export class PromptFeatures { + language: string; + length: number; + firstLineLength: number; + lastLineLength: number; + lastLineRstripLength: number; + lastLineStripLength: number; + rstripLength: number; + stripLength: number; + rstripLastLineLength: number; + rstripLastLineStripLength: number; + secondToLastLineHasComment: boolean; + rstripSecondToLastLineHasComment: boolean; + prefixEndsWithNewline: boolean; + lastChar: string; + rstripLastChar: string; + firstChar: string; + lstripFirstChar: string; + + constructor(promptComponentText: string, language: string) { + const [firstLine, lastLine] = this.firstAndLast(promptComponentText); + const firstAndLastTrimEnd = this.firstAndLast(promptComponentText.trimEnd()); + this.language = language; + this.length = promptComponentText.length; + this.firstLineLength = firstLine.length; + this.lastLineLength = lastLine.length; + this.lastLineRstripLength = lastLine.trimEnd().length; + this.lastLineStripLength = lastLine.trim().length; + this.rstripLength = promptComponentText.trimEnd().length; + this.stripLength = promptComponentText.trim().length; + this.rstripLastLineLength = firstAndLastTrimEnd[1].length; + this.rstripLastLineStripLength = firstAndLastTrimEnd[1].trim().length; + this.secondToLastLineHasComment = hasComment(promptComponentText, -2, language); + this.rstripSecondToLastLineHasComment = hasComment(promptComponentText.trimEnd(), -2, language); + this.prefixEndsWithNewline = promptComponentText.endsWith('\n'); + this.lastChar = promptComponentText.slice(-1); + this.rstripLastChar = promptComponentText.trimEnd().slice(-1); + this.firstChar = promptComponentText[0]; + this.lstripFirstChar = promptComponentText.trimStart().slice(0, 1); + } + + firstAndLast(text: string): string[] { + const lines = text.split('\n'); + const numLines = lines.length; + const firstLine = lines[0]; + let lastLine = lines[numLines - 1]; + if (lastLine === '' && numLines > 1) { + lastLine = lines[numLines - 2]; + } + return [firstLine, lastLine]; + } +} + +export class MultilineModelFeatures { + language: string; + prefixFeatures: PromptFeatures; + suffixFeatures: PromptFeatures; + + constructor(prefix: string, suffix: string, language: string) { + this.language = language; + this.prefixFeatures = new PromptFeatures(prefix, language); + this.suffixFeatures = new PromptFeatures(suffix, language); + } + + constructFeatures(): number[] { + // These features are ordered according to the features used in model training. + // They should not be reordered or revised without a corresponding change in model training. + // It is likely that not all of these features are truly necessary. However + // for now we use the same features used in the model trained by AIP for initial evaluation. + const numFeatures: number[] = new Array<number>(14).fill(0); + numFeatures[0] = this.prefixFeatures.length; + numFeatures[1] = this.prefixFeatures.firstLineLength; + numFeatures[2] = this.prefixFeatures.lastLineLength; + numFeatures[3] = this.prefixFeatures.lastLineRstripLength; + numFeatures[4] = this.prefixFeatures.lastLineStripLength; + numFeatures[5] = this.prefixFeatures.rstripLength; + numFeatures[6] = this.prefixFeatures.rstripLastLineLength; + numFeatures[7] = this.prefixFeatures.rstripLastLineStripLength; + numFeatures[8] = this.suffixFeatures.length; + numFeatures[9] = this.suffixFeatures.firstLineLength; + numFeatures[10] = this.suffixFeatures.lastLineLength; + numFeatures[11] = this.prefixFeatures.secondToLastLineHasComment ? 1 : 0; + numFeatures[12] = this.prefixFeatures.rstripSecondToLastLineHasComment ? 1 : 0; + numFeatures[13] = this.prefixFeatures.prefixEndsWithNewline ? 1 : 0; + + const langFeatures: number[] = new Array<number>(Object.keys(languageMap).length + 1).fill(0); + langFeatures[languageMap[this.language] ?? 0] = 1; + + const prefixLastCharFeatures: number[] = new Array<number>( + Object.keys(contextualFilterCharacterMap).length + 1 + ).fill(0); + prefixLastCharFeatures[contextualFilterCharacterMap[this.prefixFeatures.lastChar] ?? 0] = 1; + + const prefixRstripLastCharFeatures: number[] = new Array<number>( + Object.keys(contextualFilterCharacterMap).length + 1 + ).fill(0); + prefixRstripLastCharFeatures[contextualFilterCharacterMap[this.prefixFeatures.rstripLastChar] ?? 0] = 1; + + const suffixFirstCharFeatures: number[] = new Array<number>( + Object.keys(contextualFilterCharacterMap).length + 1 + ).fill(0); + suffixFirstCharFeatures[contextualFilterCharacterMap[this.suffixFeatures.firstChar] ?? 0] = 1; + + const suffixLstripFirstCharFeatures: number[] = new Array<number>( + Object.keys(contextualFilterCharacterMap).length + 1 + ).fill(0); + suffixLstripFirstCharFeatures[contextualFilterCharacterMap[this.suffixFeatures.lstripFirstChar] ?? 0] = 1; + + return numFeatures.concat( + langFeatures, + prefixLastCharFeatures, + prefixRstripLastCharFeatures, + suffixFirstCharFeatures, + suffixLstripFirstCharFeatures + ); + } +} + +function constructMultilineFeatures(prompt: Prompt, language: string): MultilineModelFeatures { + return new MultilineModelFeatures(prompt.prefix, prompt.suffix, language); +} + +export function requestMultilineScore(prompt: Prompt, language: string): number { + // Construct features based on the prompt and language + const features = constructMultilineFeatures(prompt, language).constructFeatures(); + // Return the score from the model which is the value at index 1 of the output array + const score = multilineModelPredict(features)[1]; + return score; +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/multilineModelWeights.ts b/completions-sample-code/vscode-node/lib/src/ghostText/multilineModelWeights.ts new file mode 100644 index 0000000..602d66a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/multilineModelWeights.ts @@ -0,0 +1,12317 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export function multilineModelPredict(input: number[]): number[] { + let var0: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 427.50000000000006) { + if (input[9] > 13.500000000000002) { + if (input[121] > 1e-35) { + var0 = -0.3793786744885956; + } else { + if (input[149] > 1e-35) { + var0 = -0.34717430705356905; + } else { + var0 = -0.26126834451035963; + } + } + } else { + var0 = -0.2431318366096852; + } + } else { + if (input[5] > 888.5000000000001) { + var0 = -0.20600463586387135; + } else { + var0 = -0.2568037008471491; + } + } + } else { + if (input[308] > 1e-35) { + var0 = -0.2363064824497454; + } else { + if (input[8] > 370.50000000000006) { + var0 = -0.37470755210284723; + } else { + var0 = -0.321978453730494; + } + } + } + } else { + if (input[3] > 24.500000000000004) { + if (input[23] > 1e-35) { + if (input[131] > 1e-35) { + var0 = -0.26259136509758885; + } else { + var0 = -0.3096719634039438; + } + } else { + if (input[4] > 30.500000000000004) { + if (input[9] > 18.500000000000004) { + var0 = -0.34254903852890883; + } else { + if (input[2] > 98.50000000000001) { + var0 = -0.41585250791146294; + } else { + var0 = -0.3673574858887241; + } + } + } else { + if (input[9] > 6.500000000000001) { + var0 = -0.31688079287876225; + } else { + if (input[31] > 1e-35) { + var0 = -0.29110977864003823; + } else { + if (input[308] > 1e-35) { + var0 = -0.3201411739040839; + } else { + var0 = -0.36874023066055506; + } + } + } + } + } + } else { + if (input[8] > 691.5000000000001) { + if (input[82] > 1e-35) { + var0 = -0.41318393149040566; + } else { + if (input[133] > 1e-35) { + var0 = -0.3741272613525161; + } else { + if (input[32] > 1e-35) { + var0 = -0.4112378041027121; + } else { + if (input[227] > 1e-35) { + var0 = -0.37726615155719356; + } else { + if (input[10] > 3.5000000000000004) { + var0 = -0.3164502293560397; + } else { + var0 = -0.2930071546509045; + } + } + } + } + } + } else { + if (input[9] > 13.500000000000002) { + var0 = -0.277366858539218; + } else { + if (input[308] > 1e-35) { + if (input[4] > 10.500000000000002) { + var0 = -0.30975610686807187; + } else { + if (input[4] > 1.5000000000000002) { + var0 = -0.2549142136728043; + } else { + var0 = -0.3271325650785176; + } + } + } else { + if (input[127] > 1e-35) { + if (input[0] > 1937.5000000000002) { + var0 = -0.2533046188098832; + } else { + var0 = -0.325520883579; + } + } else { + var0 = -0.331628896481776; + } + } + } + } + } + } + let var1: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 546.5000000000001) { + if (input[9] > 13.500000000000002) { + var1 = 0.031231253521808708; + } else { + var1 = 0.05380836288014532; + } + } else { + if (input[5] > 423.00000000000006) { + if (input[8] > 114.50000000000001) { + var1 = 0.06751619128429062; + } else { + var1 = 0.09625089153176467; + } + } else { + var1 = 0.027268163053989804; + } + } + } else { + if (input[308] > 1e-35) { + var1 = 0.060174483556283756; + } else { + var1 = -0.049062854038919135; + } + } + } else { + if (input[3] > 24.500000000000004) { + if (input[23] > 1e-35) { + if (input[4] > 63.50000000000001) { + var1 = -0.03969241799174589; + } else { + var1 = 0.01086816842550381; + } + } else { + if (input[31] > 1e-35) { + var1 = -0.003284694817583201; + } else { + if (input[9] > 6.500000000000001) { + if (input[4] > 30.500000000000004) { + var1 = -0.04224490699947552; + } else { + var1 = -0.011834162944360616; + } + } else { + if (input[308] > 1e-35) { + if (input[32] > 1e-35) { + var1 = -0.13448447971850278; + } else { + var1 = -0.019569456707046823; + } + } else { + if (input[19] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var1 = -0.07256260662659254; + } else { + if (input[4] > 60.50000000000001) { + var1 = -0.08227503453609311; + } else { + var1 = -0.020596416747563847; + } + } + } else { + var1 = -0.07396549241564149; + } + } + } + } + } + } else { + if (input[8] > 691.5000000000001) { + if (input[82] > 1e-35) { + var1 = -0.10046536995362734; + } else { + if (input[133] > 1e-35) { + var1 = -0.06407649822752297; + } else { + if (input[225] > 1e-35) { + var1 = 0.08035785003303324; + } else { + if (input[92] > 1e-35) { + var1 = 0.018901360933204676; + } else { + if (input[20] > 1e-35) { + var1 = 0.05252546973665552; + } else { + if (input[8] > 2592.5000000000005) { + var1 = -0.040543705016462955; + } else { + var1 = -0.011236043818320725; + } + } + } + } + } + } + } else { + if (input[9] > 17.500000000000004) { + var1 = 0.025560632674895334; + } else { + if (input[308] > 1e-35) { + if (input[0] > 1847.5000000000002) { + var1 = 0.03527165701669741; + } else { + var1 = -0.0071847350825815035; + } + } else { + if (input[127] > 1e-35) { + var1 = 0.024373016379595405; + } else { + if (input[9] > 2.5000000000000004) { + var1 = -0.0035090719709448288; + } else { + var1 = -0.03514829488063766; + } + } + } + } + } + } + } + let var2: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 546.5000000000001) { + var2 = 0.03848674861536988; + } else { + if (input[5] > 423.00000000000006) { + if (input[8] > 114.50000000000001) { + if (input[9] > 56.50000000000001) { + var2 = -0.003764520033319488; + } else { + var2 = 0.06570817919969299; + } + } else { + if (input[4] > 61.50000000000001) { + var2 = 0.028346156293069538; + } else { + var2 = 0.0908154644362606; + } + } + } else { + var2 = 0.02445594243234816; + } + } + } else { + if (input[308] > 1e-35) { + if (input[8] > 65.50000000000001) { + var2 = 0.0019305229020073053; + } else { + var2 = 0.09279357295883772; + } + } else { + var2 = -0.04458984161917124; + } + } + } else { + if (input[3] > 24.500000000000004) { + if (input[23] > 1e-35) { + var2 = 0.0027405390271277013; + } else { + if (input[4] > 29.500000000000004) { + if (input[52] > 1e-35) { + var2 = 0.044727478132905285; + } else { + if (input[115] > 1e-35) { + var2 = 0.10245804828855934; + } else { + if (input[9] > 17.500000000000004) { + var2 = -0.03353173647469207; + } else { + if (input[2] > 98.50000000000001) { + var2 = -0.10048106638102179; + } else { + var2 = -0.05484231104348874; + } + } + } + } + } else { + if (input[31] > 1e-35) { + var2 = 0.016807537467116516; + } else { + if (input[9] > 6.500000000000001) { + var2 = -0.012113620535295137; + } else { + if (input[4] > 8.500000000000002) { + if (input[308] > 1e-35) { + var2 = -0.01882594250504289; + } else { + var2 = -0.05585658862796076; + } + } else { + var2 = 0.04279591277938338; + } + } + } + } + } + } else { + if (input[8] > 691.5000000000001) { + if (input[82] > 1e-35) { + var2 = -0.09262278043707878; + } else { + if (input[133] > 1e-35) { + var2 = -0.058454257768893625; + } else { + if (input[32] > 1e-35) { + var2 = -0.09769348447126434; + } else { + if (input[25] > 1e-35) { + var2 = -0.0725430043727677; + } else { + if (input[122] > 1e-35) { + var2 = -0.10047841601578077; + } else { + var2 = -0.00580671054458958; + } + } + } + } + } + } else { + if (input[9] > 13.500000000000002) { + var2 = 0.021399199032818294; + } else { + if (input[308] > 1e-35) { + if (input[4] > 10.500000000000002) { + var2 = -0.0076376731757173515; + } else { + var2 = 0.03394923033036848; + } + } else { + if (input[127] > 1e-35) { + var2 = 0.02070489091204209; + } else { + var2 = -0.02290162726126496; + } + } + } + } + } + } + let var3: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 892.5000000000001) { + if (input[9] > 21.500000000000004) { + var3 = 0.010230295672324606; + } else { + var3 = 0.038540509248742805; + } + } else { + if (input[8] > 125.50000000000001) { + if (input[1] > 49.50000000000001) { + var3 = 0.03086356292895467; + } else { + var3 = 0.057128750867458604; + } + } else { + if (input[5] > 888.5000000000001) { + var3 = 0.07861602941396924; + } else { + var3 = 0.030523262699070908; + } + } + } + } else { + if (input[308] > 1e-35) { + var3 = 0.048236117667577356; + } else { + if (input[8] > 370.50000000000006) { + var3 = -0.05642125069212264; + } else { + var3 = -0.007232836777168195; + } + } + } + } else { + if (input[3] > 24.500000000000004) { + if (input[23] > 1e-35) { + if (input[131] > 1e-35) { + var3 = 0.03640661467213915; + } else { + var3 = -0.005889820723907028; + } + } else { + if (input[31] > 1e-35) { + var3 = -0.0009007166998276938; + } else { + if (input[9] > 6.500000000000001) { + var3 = -0.022590340093882378; + } else { + if (input[308] > 1e-35) { + if (input[32] > 1e-35) { + var3 = -0.1215445089091064; + } else { + var3 = -0.01435612266219722; + } + } else { + if (input[19] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var3 = -0.061555513040777825; + } else { + if (input[4] > 60.50000000000001) { + var3 = -0.07053475504569347; + } else { + var3 = -0.013733369453963092; + } + } + } else { + var3 = -0.06302097189114152; + } + } + } + } + } + } else { + if (input[227] > 1e-35) { + var3 = -0.05820440333190048; + } else { + if (input[8] > 683.5000000000001) { + if (input[82] > 1e-35) { + var3 = -0.08466979526809346; + } else { + if (input[10] > 24.500000000000004) { + var3 = -0.017092159721119944; + } else { + if (input[92] > 1e-35) { + var3 = 0.03592901452463749; + } else { + var3 = -0.00359310519524756; + } + } + } + } else { + if (input[5] > 1809.5000000000002) { + if (input[243] > 1e-35) { + var3 = -0.03963116207386097; + } else { + if (input[118] > 1e-35) { + var3 = -0.09483996283536394; + } else { + if (input[217] > 1e-35) { + var3 = -0.03394542089519989; + } else { + if (input[242] > 1e-35) { + var3 = -0.07985899422287938; + } else { + var3 = 0.019706602160656964; + } + } + } + } + } else { + if (input[9] > 12.500000000000002) { + var3 = 0.014072998937735146; + } else { + var3 = -0.021156294523894684; + } + } + } + } + } + } + let var4: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 892.5000000000001) { + if (input[9] > 21.500000000000004) { + var4 = 0.009197756540516563; + } else { + var4 = 0.03458896869535166; + } + } else { + if (input[5] > 5082.500000000001) { + var4 = 0.08265545468131008; + } else { + if (input[131] > 1e-35) { + var4 = 0.0740738432473315; + } else { + var4 = 0.045159136632942756; + } + } + } + } else { + if (input[8] > 319.50000000000006) { + var4 = -0.04653401534465376; + } else { + if (input[7] > 3.5000000000000004) { + if (input[0] > 1230.5000000000002) { + if (input[0] > 2579.5000000000005) { + var4 = -0.011400839766681709; + } else { + var4 = 0.11149800187510031; + } + } else { + var4 = -0.08683250977599462; + } + } else { + var4 = 0.08355310136724753; + } + } + } + } else { + if (input[4] > 23.500000000000004) { + if (input[23] > 1e-35) { + if (input[131] > 1e-35) { + var4 = 0.040389083779932555; + } else { + var4 = -0.009887614274108602; + } + } else { + if (input[52] > 1e-35) { + var4 = 0.03705353499757327; + } else { + if (input[9] > 6.500000000000001) { + var4 = -0.025401260429257562; + } else { + if (input[2] > 98.50000000000001) { + var4 = -0.09237673187534504; + } else { + var4 = -0.04298556869281803; + } + } + } + } + } else { + if (input[222] > 1e-35) { + var4 = -0.045221965895986184; + } else { + if (input[8] > 691.5000000000001) { + if (input[133] > 1e-35) { + var4 = -0.05435318330148897; + } else { + if (input[128] > 1e-35) { + var4 = -0.08672907303184191; + } else { + if (input[227] > 1e-35) { + var4 = -0.05568304584186561; + } else { + if (input[122] > 1e-35) { + var4 = -0.09623059693538563; + } else { + if (input[225] > 1e-35) { + var4 = 0.07558331642202279; + } else { + if (input[82] > 1e-35) { + var4 = -0.07360566227233566; + } else { + var4 = -0.005646164647395919; + } + } + } + } + } + } + } else { + if (input[242] > 1e-35) { + var4 = -0.08203758341228108; + } else { + if (input[9] > 13.500000000000002) { + var4 = 0.018726123829696042; + } else { + if (input[308] > 1e-35) { + if (input[4] > 10.500000000000002) { + var4 = -0.011153942154062704; + } else { + var4 = 0.03132858912391067; + } + } else { + if (input[127] > 1e-35) { + var4 = 0.021455228822345174; + } else { + if (input[23] > 1e-35) { + var4 = 0.01959966745346997; + } else { + var4 = -0.021764790177579325; + } + } + } + } + } + } + } + } + } + let var5: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 284.50000000000006) { + if (input[121] > 1e-35) { + if (input[18] > 1e-35) { + var5 = 0.07547602514276922; + } else { + var5 = -0.08529678832140396; + } + } else { + var5 = 0.030314822344598043; + } + } else { + if (input[5] > 888.5000000000001) { + if (input[4] > 61.50000000000001) { + var5 = 0.011143589009415464; + } else { + var5 = 0.0654700456802118; + } + } else { + var5 = 0.021794712646632755; + } + } + } else { + if (input[308] > 1e-35) { + var5 = 0.04231872551095028; + } else { + var5 = -0.034381999950549455; + } + } + } else { + if (input[4] > 23.500000000000004) { + if (input[23] > 1e-35) { + if (input[4] > 63.50000000000001) { + var5 = -0.03678981254332261; + } else { + var5 = 0.010518160384496255; + } + } else { + if (input[8] > 825.5000000000001) { + var5 = -0.04506534842082387; + } else { + if (input[9] > 38.50000000000001) { + var5 = 0.01004983052203438; + } else { + var5 = -0.030580958620701027; + } + } + } + } else { + if (input[39] > 1e-35) { + var5 = -0.12802435021505382; + } else { + if (input[8] > 691.5000000000001) { + if (input[23] > 1e-35) { + if (input[203] > 1e-35) { + if (input[4] > 6.500000000000001) { + var5 = 0.030426957004611704; + } else { + var5 = -0.0726407693060581; + } + } else { + var5 = 0.017395521646964375; + } + } else { + if (input[4] > 7.500000000000001) { + if (input[0] > 93.50000000000001) { + if (input[9] > 7.500000000000001) { + var5 = -0.008024349629981291; + } else { + if (input[31] > 1e-35) { + var5 = 0.01296539930850471; + } else { + if (input[308] > 1e-35) { + var5 = -0.012855016509024084; + } else { + var5 = -0.04564527976851505; + } + } + } + } else { + var5 = -0.15681420504058596; + } + } else { + if (input[10] > 4.500000000000001) { + if (input[243] > 1e-35) { + var5 = -0.1012064426380198; + } else { + var5 = -0.0062808850924854194; + } + } else { + var5 = 0.030706323726162416; + } + } + } + } else { + if (input[9] > 13.500000000000002) { + var5 = 0.017081636133736405; + } else { + if (input[308] > 1e-35) { + if (input[4] > 10.500000000000002) { + var5 = -0.009306613091760644; + } else { + if (input[4] > 1.5000000000000002) { + var5 = 0.03655523200850989; + } else { + var5 = -0.02671654212893341; + } + } + } else { + if (input[127] > 1e-35) { + var5 = 0.019261510468604387; + } else { + var5 = -0.017627818570628936; + } + } + } + } + } + } + } + let var6: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 892.5000000000001) { + if (input[308] > 1e-35) { + var6 = 0.036100405995889276; + } else { + var6 = 0.011709313297015793; + } + } else { + if (input[0] > 119.50000000000001) { + if (input[8] > 125.50000000000001) { + var6 = 0.03622542297472574; + } else { + var6 = 0.05595579157301536; + } + } else { + var6 = -0.02234751038146796; + } + } + } else { + if (input[8] > 319.50000000000006) { + var6 = -0.040132029478400735; + } else { + if (input[7] > 3.5000000000000004) { + if (input[0] > 1230.5000000000002) { + if (input[0] > 2579.5000000000005) { + var6 = -0.009306153573847916; + } else { + var6 = 0.10058509567064988; + } + } else { + var6 = -0.0785668890966017; + } + } else { + if (input[9] > 28.500000000000004) { + var6 = -0.04781977604130416; + } else { + var6 = 0.09753292614937459; + } + } + } + } + } else { + if (input[4] > 23.500000000000004) { + if (input[131] > 1e-35) { + var6 = 0.02372493254975127; + } else { + if (input[148] > 1e-35) { + var6 = 0.028103095989516644; + } else { + if (input[4] > 58.50000000000001) { + if (input[10] > 1e-35) { + var6 = -0.05000852203469597; + } else { + var6 = 0.02922366846119705; + } + } else { + if (input[23] > 1e-35) { + var6 = -0.0026335076988151292; + } else { + var6 = -0.03073993752935585; + } + } + } + } + } else { + if (input[222] > 1e-35) { + var6 = -0.03867374428185713; + } else { + if (input[32] > 1e-35) { + var6 = -0.07220729365053084; + } else { + if (input[39] > 1e-35) { + var6 = -0.11624524614351733; + } else { + if (input[8] > 691.5000000000001) { + if (input[133] > 1e-35) { + var6 = -0.04836360271198036; + } else { + if (input[8] > 4968.500000000001) { + var6 = -0.10873681915578029; + } else { + if (input[149] > 1e-35) { + var6 = -0.11847484033769298; + } else { + if (input[122] > 1e-35) { + var6 = -0.08916172460307559; + } else { + if (input[82] > 1e-35) { + var6 = -0.06774726602152634; + } else { + var6 = -0.0033469147714351327; + } + } + } + } + } + } else { + if (input[126] > 1e-35) { + var6 = -0.09474445392080015; + } else { + if (input[8] > 131.50000000000003) { + if (input[118] > 1e-35) { + var6 = -0.09002547031023511; + } else { + var6 = 0.015475385187009489; + } + } else { + if (input[25] > 1e-35) { + var6 = -0.08175501232759151; + } else { + var6 = -0.000429679055394914; + } + } + } + } + } + } + } + } + } + let var7: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 546.5000000000001) { + var7 = 0.021942996005324917; + } else { + var7 = 0.042349138084484074; + } + } else { + if (input[308] > 1e-35) { + var7 = 0.036507270845732874; + } else { + var7 = -0.028981850556764995; + } + } + } else { + if (input[3] > 24.500000000000004) { + if (input[23] > 1e-35) { + var7 = 0.00210930790963475; + } else { + if (input[31] > 1e-35) { + var7 = 0.006825358293027163; + } else { + if (input[9] > 6.500000000000001) { + var7 = -0.013772084269062394; + } else { + if (input[308] > 1e-35) { + var7 = -0.008307929099892574; + } else { + if (input[19] > 1e-35) { + var7 = -0.027706313312904487; + } else { + var7 = -0.04891108984170914; + } + } + } + } + } + } else { + if (input[134] > 1e-35) { + var7 = -0.0605730733844732; + } else { + if (input[25] > 1e-35) { + var7 = -0.05347926493253117; + } else { + if (input[227] > 1e-35) { + var7 = -0.049415829249003666; + } else { + if (input[32] > 1e-35) { + var7 = -0.06807799662179595; + } else { + if (input[308] > 1e-35) { + if (input[4] > 10.500000000000002) { + if (input[2] > 13.500000000000002) { + var7 = -0.00016302718260794637; + } else { + var7 = -0.10247095758122947; + } + } else { + if (input[210] > 1e-35) { + var7 = -0.022149002072787024; + } else { + if (input[95] > 1e-35) { + var7 = 0.15222631630626304; + } else { + var7 = 0.027393884520465712; + } + } + } + } else { + if (input[9] > 7.500000000000001) { + if (input[225] > 1e-35) { + var7 = 0.13483346577752245; + } else { + if (input[3] > 9.500000000000002) { + if (input[243] > 1e-35) { + var7 = -0.045352728133789516; + } else { + if (input[8] > 683.5000000000001) { + var7 = 0.00474372227519902; + } else { + var7 = 0.02635476098707525; + } + } + } else { + if (input[92] > 1e-35) { + var7 = 0.05659380819933452; + } else { + if (input[105] > 1e-35) { + var7 = 0.07431443210341222; + } else { + if (input[186] > 1e-35) { + var7 = 0.0915821133384904; + } else { + var7 = -0.016414750130401053; + } + } + } + } + } + } else { + if (input[127] > 1e-35) { + var7 = 0.011824693641866162; + } else { + if (input[23] > 1e-35) { + var7 = 0.0228468674288774; + } else { + if (input[284] > 1e-35) { + var7 = 0.06606936863302432; + } else { + var7 = -0.02872463273902358; + } + } + } + } + } + } + } + } + } + } + } + let var8: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[8] > 125.50000000000001) { + if (input[288] > 1e-35) { + var8 = -0.019844363904157558; + } else { + if (input[1] > 50.50000000000001) { + if (input[131] > 1e-35) { + var8 = 0.044961338592245194; + } else { + var8 = 0.003659599513761676; + } + } else { + if (input[121] > 1e-35) { + var8 = -0.04057103630479994; + } else { + var8 = 0.03158560697078578; + } + } + } + } else { + if (input[0] > 421.50000000000006) { + if (input[4] > 61.50000000000001) { + var8 = -0.0003708603406529278; + } else { + var8 = 0.05331312264472391; + } + } else { + var8 = 0.0006575958601218936; + } + } + } else { + if (input[8] > 319.50000000000006) { + var8 = -0.034654694051901545; + } else { + if (input[7] > 3.5000000000000004) { + if (input[0] > 1230.5000000000002) { + if (input[0] > 2579.5000000000005) { + var8 = -0.0076053515916517005; + } else { + var8 = 0.09116695486305336; + } + } else { + var8 = -0.07137458699162028; + } + } else { + var8 = 0.06633130654035282; + } + } + } + } else { + if (input[4] > 29.500000000000004) { + if (input[23] > 1e-35) { + if (input[4] > 63.50000000000001) { + var8 = -0.0308520802187302; + } else { + var8 = 0.013156423968295541; + } + } else { + if (input[115] > 1e-35) { + var8 = 0.11581171687488252; + } else { + if (input[52] > 1e-35) { + if (input[10] > 22.500000000000004) { + var8 = 0.12264179915175587; + } else { + var8 = -0.021905727233873535; + } + } else { + if (input[8] > 799.5000000000001) { + var8 = -0.04181869575935412; + } else { + var8 = -0.023695901673350575; + } + } + } + } + } else { + if (input[222] > 1e-35) { + var8 = -0.034612899265371776; + } else { + if (input[8] > 691.5000000000001) { + if (input[9] > 98.50000000000001) { + var8 = -0.06892116536821917; + } else { + if (input[149] > 1e-35) { + var8 = -0.11194586444154514; + } else { + if (input[133] > 1e-35) { + var8 = -0.04269583234000504; + } else { + if (input[128] > 1e-35) { + var8 = -0.0644631966969502; + } else { + if (input[8] > 4968.500000000001) { + var8 = -0.09650726096330133; + } else { + var8 = -0.004219129180139438; + } + } + } + } + } + } else { + if (input[126] > 1e-35) { + var8 = -0.08038306745347751; + } else { + if (input[5] > 1809.5000000000002) { + var8 = 0.009265335288169993; + } else { + if (input[9] > 2.5000000000000004) { + var8 = 0.006447645462117438; + } else { + var8 = -0.021047132609551503; + } + } + } + } + } + } + } + let var9: number; + if (input[13] > 1e-35) { + if (input[3] > 1.5000000000000002) { + if (input[9] > 21.500000000000004) { + if (input[121] > 1e-35) { + var9 = -0.08436540015142402; + } else { + if (input[8] > 1861.5000000000002) { + var9 = -0.01621425699342421; + } else { + var9 = 0.01878613821895428; + } + } + } else { + var9 = 0.031052879158242532; + } + } else { + if (input[8] > 319.50000000000006) { + var9 = -0.031536619360997865; + } else { + if (input[7] > 3.5000000000000004) { + var9 = -0.004510586962343298; + } else { + var9 = 0.0596524941011746; + } + } + } + } else { + if (input[4] > 18.500000000000004) { + if (input[23] > 1e-35) { + var9 = 0.004757490541310808; + } else { + if (input[9] > 6.500000000000001) { + var9 = -0.008842393772207996; + } else { + if (input[31] > 1e-35) { + var9 = 0.0010536183837006993; + } else { + if (input[308] > 1e-35) { + var9 = -0.008145882815435419; + } else { + if (input[2] > 98.50000000000001) { + var9 = -0.08404937622173021; + } else { + if (input[276] > 1e-35) { + var9 = 0.0020072791321856663; + } else { + if (input[19] > 1e-35) { + var9 = -0.023031820639490178; + } else { + var9 = -0.04553314326377875; + } + } + } + } + } + } + } + } else { + if (input[8] > 2134.5000000000005) { + var9 = -0.02244583113572251; + } else { + if (input[134] > 1e-35) { + var9 = -0.05592137394753121; + } else { + if (input[308] > 1e-35) { + if (input[49] > 1e-35) { + var9 = 0.09989109704064947; + } else { + if (input[4] > 10.500000000000002) { + if (input[2] > 13.500000000000002) { + var9 = -0.00447733056482096; + } else { + var9 = -0.10191061664873849; + } + } else { + var9 = 0.021765308380331864; + } + } + } else { + if (input[9] > 7.500000000000001) { + if (input[118] > 1e-35) { + var9 = -0.07570059131536411; + } else { + if (input[243] > 1e-35) { + var9 = -0.040983393346598646; + } else { + if (input[3] > 9.500000000000002) { + var9 = 0.014763759061483812; + } else { + if (input[92] > 1e-35) { + var9 = 0.05136368898963024; + } else { + var9 = -0.008162398981149495; + } + } + } + } + } else { + if (input[127] > 1e-35) { + var9 = 0.013999119696708346; + } else { + if (input[23] > 1e-35) { + if (input[20] > 1e-35) { + var9 = 0.14138985500120907; + } else { + var9 = 0.008668274102844162; + } + } else { + if (input[284] > 1e-35) { + var9 = 0.06356484011042893; + } else { + var9 = -0.024781304572706303; + } + } + } + } + } + } + } + } + } + let var10: number; + if (input[13] > 1e-35) { + if (input[3] > 8.500000000000002) { + if (input[8] > 892.5000000000001) { + if (input[0] > 384.50000000000006) { + var10 = 0.014387526569215037; + } else { + if (input[8] > 2266.5000000000005) { + var10 = -0.1397298649743087; + } else { + var10 = 0.007953931014097788; + } + } + } else { + if (input[0] > 119.50000000000001) { + if (input[4] > 61.50000000000001) { + var10 = 0.0029819092211896296; + } else { + if (input[218] > 1e-35) { + var10 = 0.08450459375645737; + } else { + var10 = 0.031646488019280654; + } + } + } else { + var10 = -0.03544960151460596; + } + } + } else { + if (input[9] > 9.500000000000002) { + var10 = -0.026002317735915183; + } else { + if (input[7] > 1.5000000000000002) { + var10 = 0.005074258810794793; + } else { + var10 = 0.0745247650477651; + } + } + } + } else { + if (input[4] > 29.500000000000004) { + if (input[131] > 1e-35) { + var10 = 0.023269218675640847; + } else { + if (input[148] > 1e-35) { + var10 = 0.03812942399144545; + } else { + if (input[115] > 1e-35) { + var10 = 0.10512283476967227; + } else { + var10 = -0.02607307479736138; + } + } + } + } else { + if (input[227] > 1e-35) { + var10 = -0.036576708299046294; + } else { + if (input[101] > 1e-35) { + var10 = 0.027948683650881864; + } else { + if (input[149] > 1e-35) { + var10 = -0.08195628451594297; + } else { + if (input[50] > 1e-35) { + var10 = -0.16997544922278504; + } else { + if (input[8] > 691.5000000000001) { + if (input[9] > 101.50000000000001) { + var10 = -0.06860333850762075; + } else { + if (input[225] > 1e-35) { + var10 = 0.06066641950951723; + } else { + if (input[10] > 22.500000000000004) { + if (input[1] > 29.500000000000004) { + if (input[127] > 1e-35) { + var10 = 0.028599705845427533; + } else { + var10 = -0.010746719511640914; + } + } else { + if (input[0] > 4877.500000000001) { + var10 = -0.07251187886096228; + } else { + var10 = -0.021299712241446785; + } + } + } else { + if (input[118] > 1e-35) { + var10 = -0.11902023760964736; + } else { + var10 = 0.000015874469526809387; + } + } + } + } + } else { + if (input[8] > 267.50000000000006) { + var10 = 0.01317292185402293; + } else { + if (input[148] > 1e-35) { + if (input[9] > 20.500000000000004) { + var10 = 0.09614842415142123; + } else { + var10 = 0.006049073167176467; + } + } else { + if (input[189] > 1e-35) { + var10 = 0.05562696451900713; + } else { + var10 = -0.006257541923837303; + } + } + } + } + } + } + } + } + } + } + let var11: number; + if (input[13] > 1e-35) { + if (input[9] > 14.500000000000002) { + if (input[2] > 11.500000000000002) { + if (input[1] > 71.50000000000001) { + if (input[8] > 1252.5000000000002) { + var11 = -0.10069846585436666; + } else { + var11 = -0.010577995535809317; + } + } else { + if (input[146] > 1e-35) { + var11 = -0.008877238274428668; + } else { + if (input[280] > 1e-35) { + var11 = 0.10076055897012692; + } else { + if (input[6] > 70.50000000000001) { + var11 = -0.020603523042565547; + } else { + if (input[7] > 1.5000000000000002) { + var11 = 0.02819095420813202; + } else { + var11 = -0.1223354167911277; + } + } + } + } + } + } else { + var11 = -0.025073583348334844; + } + } else { + if (input[8] > 416.50000000000006) { + var11 = 0.01718560189149466; + } else { + if (input[230] > 1e-35) { + var11 = 0.12281803224342265; + } else { + var11 = 0.03281276971308565; + } + } + } + } else { + if (input[4] > 14.500000000000002) { + if (input[23] > 1e-35) { + if (input[21] > 1e-35) { + var11 = -0.13070568109867683; + } else { + if (input[4] > 63.50000000000001) { + var11 = -0.027221825262496814; + } else { + var11 = 0.01530862490082352; + } + } + } else { + if (input[9] > 6.500000000000001) { + if (input[5] > 4320.500000000001) { + if (input[2] > 31.500000000000004) { + var11 = -0.00605574271293711; + } else { + var11 = 0.04739407327741249; + } + } else { + var11 = -0.012537528620315956; + } + } else { + if (input[31] > 1e-35) { + if (input[20] > 1e-35) { + var11 = 0.1252215087035768; + } else { + var11 = 0.003905888677601057; + } + } else { + if (input[52] > 1e-35) { + var11 = 0.045466299731038815; + } else { + if (input[2] > 100.50000000000001) { + var11 = -0.07815624550168065; + } else { + if (input[308] > 1e-35) { + var11 = -0.007715815250508057; + } else { + if (input[276] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var11 = -0.03538265083203445; + } else { + if (input[18] > 1e-35) { + var11 = 0.1591211669800727; + } else { + var11 = 0.015151475408241136; + } + } + } else { + if (input[8] > 557.5000000000001) { + var11 = -0.04225569725456342; + } else { + var11 = -0.022455546324243267; + } + } + } + } + } + } + } + } + } else { + if (input[308] > 1e-35) { + var11 = 0.01325441736085826; + } else { + if (input[197] > 1e-35) { + var11 = 0.03752194600682512; + } else { + if (input[225] > 1e-35) { + var11 = 0.06583712394533976; + } else { + var11 = -0.005205289866839043; + } + } + } + } + } + let var12: number; + if (input[13] > 1e-35) { + if (input[9] > 21.500000000000004) { + if (input[2] > 12.500000000000002) { + var12 = 0.010264022580774884; + } else { + var12 = -0.02335958814489217; + } + } else { + if (input[8] > 416.50000000000006) { + if (input[3] > 4.500000000000001) { + if (input[295] > 1e-35) { + var12 = -0.0936747137352166; + } else { + if (input[0] > 384.50000000000006) { + var12 = 0.019846244507320695; + } else { + var12 = -0.0751102554077272; + } + } + } else { + var12 = -0.026885329334203723; + } + } else { + if (input[0] > 966.5000000000001) { + if (input[10] > 48.50000000000001) { + var12 = 0.11654906890054273; + } else { + var12 = 0.0346250587613322; + } + } else { + if (input[4] > 39.50000000000001) { + var12 = -0.08568002378645614; + } else { + if (input[9] > 16.500000000000004) { + var12 = -0.12010535752923689; + } else { + var12 = 0.021321923389033808; + } + } + } + } + } + } else { + if (input[4] > 14.500000000000002) { + if (input[23] > 1e-35) { + if (input[21] > 1e-35) { + var12 = -0.12056431231412057; + } else { + if (input[131] > 1e-35) { + var12 = 0.03652965550568472; + } else { + var12 = 0.002563006128791669; + } + } + } else { + if (input[9] > 6.500000000000001) { + if (input[30] > 1e-35) { + var12 = -0.10141481732178981; + } else { + var12 = -0.003936457893178248; + } + } else { + if (input[31] > 1e-35) { + var12 = 0.008215898756249477; + } else { + if (input[52] > 1e-35) { + if (input[0] > 4188.500000000001) { + var12 = 0.12972828769588213; + } else { + var12 = -0.003137412232297087; + } + } else { + if (input[2] > 100.50000000000001) { + var12 = -0.0730872929087944; + } else { + if (input[308] > 1e-35) { + var12 = -0.006958622747243333; + } else { + if (input[35] > 1e-35) { + if (input[0] > 3707.5000000000005) { + var12 = 0.07934620723812878; + } else { + var12 = -0.018598568353702116; + } + } else { + var12 = -0.030635505446410763; + } + } + } + } + } + } + } + } else { + if (input[128] > 1e-35) { + var12 = -0.06962290453843294; + } else { + if (input[84] > 1e-35) { + var12 = -0.15290337844960322; + } else { + if (input[308] > 1e-35) { + if (input[8] > 2543.5000000000005) { + var12 = -0.034938657503885584; + } else { + var12 = 0.016339322898966915; + } + } else { + if (input[197] > 1e-35) { + var12 = 0.03358907965870046; + } else { + if (input[18] > 1e-35) { + var12 = -0.01754013791515288; + } else { + var12 = -0.0004944586067698557; + } + } + } + } + } + } + } + let var13: number; + if (input[13] > 1e-35) { + if (input[308] > 1e-35) { + if (input[210] > 1e-35) { + var13 = 0.005888790687820524; + } else { + var13 = 0.0429676533834978; + } + } else { + if (input[2] > 7.500000000000001) { + if (input[0] > 119.50000000000001) { + if (input[6] > 79.50000000000001) { + var13 = -0.0224319889201976; + } else { + if (input[212] > 1e-35) { + var13 = 0.06249587051783863; + } else { + if (input[8] > 963.5000000000001) { + if (input[8] > 1156.5000000000002) { + var13 = 0.010357273289123324; + } else { + var13 = -0.029749145161304082; + } + } else { + if (input[218] > 1e-35) { + var13 = 0.06449336340743606; + } else { + var13 = 0.018047654539345502; + } + } + } + } + } else { + var13 = -0.07350502390293116; + } + } else { + var13 = -0.019594829995832414; + } + } + } else { + if (input[4] > 39.50000000000001) { + var13 = -0.019338083179859314; + } else { + if (input[39] > 1e-35) { + var13 = -0.10427066919173111; + } else { + if (input[222] > 1e-35) { + if (input[0] > 612.5000000000001) { + var13 = -0.019197415255018464; + } else { + var13 = -0.0836562507048181; + } + } else { + if (input[149] > 1e-35) { + var13 = -0.07679624472577429; + } else { + if (input[32] > 1e-35) { + var13 = -0.05097506748590604; + } else { + if (input[191] > 1e-35) { + var13 = 0.04670476485250936; + } else { + if (input[30] > 1e-35) { + var13 = -0.05313073892148652; + } else { + if (input[8] > 691.5000000000001) { + if (input[23] > 1e-35) { + if (input[203] > 1e-35) { + if (input[4] > 8.500000000000002) { + var13 = 0.03930363008271334; + } else { + var13 = -0.06029171685615689; + } + } else { + var13 = 0.016203086182431294; + } + } else { + if (input[4] > 7.500000000000001) { + var13 = -0.013824248237085224; + } else { + if (input[10] > 4.500000000000001) { + if (input[94] > 1e-35) { + var13 = -0.09817668643367765; + } else { + if (input[10] > 40.50000000000001) { + var13 = -0.023558078753593125; + } else { + var13 = 0.0065113494780482326; + } + } + } else { + if (input[8] > 809.5000000000001) { + if (input[297] > 1e-35) { + var13 = -0.1352063548573715; + } else { + var13 = 0.058203900441270634; + } + } else { + var13 = -0.035243959159285736; + } + } + } + } + } else { + if (input[10] > 59.50000000000001) { + if (input[1] > 43.50000000000001) { + var13 = -0.012552876807800442; + } else { + var13 = 0.05991247777734298; + } + } else { + var13 = 0.0035893102109330177; + } + } + } + } + } + } + } + } + } + } + let var14: number; + if (input[13] > 1e-35) { + if (input[9] > 21.500000000000004) { + if (input[145] > 1e-35) { + var14 = 0.03507251990078782; + } else { + if (input[2] > 14.500000000000002) { + var14 = 0.004905698363309292; + } else { + if (input[8] > 2421.5000000000005) { + var14 = -0.10306119951984316; + } else { + var14 = -0.018951037816654928; + } + } + } + } else { + if (input[8] > 416.50000000000006) { + if (input[3] > 4.500000000000001) { + if (input[295] > 1e-35) { + var14 = -0.08503171085833393; + } else { + var14 = 0.015130974593044409; + } + } else { + var14 = -0.024425267075198206; + } + } else { + var14 = 0.02624054905103126; + } + } + } else { + if (input[4] > 19.500000000000004) { + if (input[131] > 1e-35) { + var14 = 0.02100191580704534; + } else { + if (input[32] > 1e-35) { + if (input[8] > 2302.5000000000005) { + var14 = 0.09908783187786288; + } else { + var14 = -0.06920877329925636; + } + } else { + if (input[8] > 241.50000000000003) { + var14 = -0.016756131804203496; + } else { + if (input[9] > 33.50000000000001) { + var14 = 0.04903179955263626; + } else { + if (input[217] > 1e-35) { + var14 = -0.047416847619291644; + } else { + var14 = -0.0017200891991431119; + } + } + } + } + } + } else { + if (input[39] > 1e-35) { + var14 = -0.10389927604977028; + } else { + if (input[134] > 1e-35) { + var14 = -0.050480365434872866; + } else { + if (input[178] > 1e-35) { + var14 = -0.05167855791556937; + } else { + if (input[8] > 2134.5000000000005) { + var14 = -0.01663197335585307; + } else { + if (input[242] > 1e-35) { + var14 = -0.05361323756615453; + } else { + if (input[118] > 1e-35) { + var14 = -0.05299780866211368; + } else { + if (input[10] > 24.500000000000004) { + if (input[10] > 55.50000000000001) { + if (input[8] > 764.5000000000001) { + var14 = -0.0016544848369620534; + } else { + var14 = 0.04494144460483587; + } + } else { + var14 = -0.009283616456736156; + } + } else { + if (input[121] > 1e-35) { + if (input[0] > 4463.500000000001) { + var14 = 0.051166688553608355; + } else { + var14 = -0.06623908820705383; + } + } else { + if (input[84] > 1e-35) { + var14 = -0.12990936092409747; + } else { + if (input[306] > 1e-35) { + var14 = -0.07020596855118943; + } else { + if (input[49] > 1e-35) { + var14 = 0.06272964802556856; + } else { + if (input[192] > 1e-35) { + var14 = 0.06540204627162581; + } else { + var14 = 0.008277910531592885; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + let var15: number; + if (input[13] > 1e-35) { + if (input[308] > 1e-35) { + if (input[210] > 1e-35) { + var15 = 0.003325460510319164; + } else { + var15 = 0.037153108286272905; + } + } else { + if (input[2] > 12.500000000000002) { + if (input[1] > 124.50000000000001) { + var15 = -0.09880713344892134; + } else { + if (input[7] > 60.50000000000001) { + if (input[10] > 71.50000000000001) { + var15 = 0.0697359767152808; + } else { + if (input[230] > 1e-35) { + var15 = 0.06513506845651572; + } else { + var15 = -0.02826625276613455; + } + } + } else { + if (input[5] > 246.50000000000003) { + if (input[8] > 95.50000000000001) { + var15 = 0.013616385013146277; + } else { + var15 = 0.04171540100223404; + } + } else { + var15 = -0.04360396575094823; + } + } + } + } else { + if (input[212] > 1e-35) { + var15 = 0.025945477945627522; + } else { + var15 = -0.019793208261535442; + } + } + } + } else { + if (input[4] > 39.50000000000001) { + if (input[25] > 1e-35) { + var15 = -0.07856453318384411; + } else { + var15 = -0.014803893522351739; + } + } else { + if (input[39] > 1e-35) { + var15 = -0.09185452630751932; + } else { + if (input[149] > 1e-35) { + var15 = -0.07122426086157027; + } else { + if (input[134] > 1e-35) { + var15 = -0.04231052091434186; + } else { + if (input[227] > 1e-35) { + var15 = -0.029815824273994197; + } else { + if (input[50] > 1e-35) { + var15 = -0.15736496271211153; + } else { + if (input[222] > 1e-35) { + var15 = -0.02360285356956629; + } else { + if (input[128] > 1e-35) { + var15 = -0.03922080193836443; + } else { + if (input[136] > 1e-35) { + var15 = -0.07219685327698587; + } else { + if (input[10] > 24.500000000000004) { + if (input[1] > 8.500000000000002) { + var15 = -0.0029736170756835783; + } else { + var15 = -0.06482902102259112; + } + } else { + if (input[84] > 1e-35) { + var15 = -0.11340924635708383; + } else { + if (input[94] > 1e-35) { + var15 = -0.03635703457792193; + } else { + if (input[118] > 1e-35) { + var15 = -0.058181913914186034; + } else { + if (input[126] > 1e-35) { + var15 = -0.062030576241517366; + } else { + if (input[116] > 1e-35) { + var15 = -0.045086301850604006; + } else { + if (input[25] > 1e-35) { + var15 = -0.031665223656767286; + } else { + if (input[203] > 1e-35) { + var15 = -0.009444685731407691; + } else { + var15 = 0.0112265153772187; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + let var16: number; + if (input[13] > 1e-35) { + if (input[1] > 64.50000000000001) { + if (input[9] > 14.500000000000002) { + if (input[9] > 54.50000000000001) { + var16 = 0.022717227245241684; + } else { + var16 = -0.049700413274686266; + } + } else { + var16 = 0.007175776918589741; + } + } else { + if (input[5] > 50.50000000000001) { + if (input[8] > 61.50000000000001) { + if (input[21] > 1e-35) { + var16 = -0.07927556792063156; + } else { + if (input[3] > 8.500000000000002) { + if (input[4] > 23.500000000000004) { + if (input[281] > 1e-35) { + var16 = -0.12263724050601095; + } else { + var16 = 0.0070743478891288035; + } + } else { + if (input[288] > 1e-35) { + var16 = -0.050439138582109; + } else { + var16 = 0.0255701593657891; + } + } + } else { + var16 = -0.005812703740580558; + } + } + } else { + if (input[6] > 49.50000000000001) { + var16 = -0.008542694147899113; + } else { + var16 = 0.035147383686665; + } + } + } else { + var16 = -0.0960461939274094; + } + } + } else { + if (input[32] > 1e-35) { + var16 = -0.04555453745517765; + } else { + if (input[222] > 1e-35) { + if (input[0] > 612.5000000000001) { + var16 = -0.01800870272656664; + } else { + var16 = -0.07817304234604389; + } + } else { + if (input[30] > 1e-35) { + var16 = -0.05227061750368981; + } else { + if (input[25] > 1e-35) { + if (input[0] > 4449.500000000001) { + if (input[217] > 1e-35) { + var16 = 0.08778416018479411; + } else { + var16 = -0.026563982720830256; + } + } else { + var16 = -0.05296139548112329; + } + } else { + if (input[50] > 1e-35) { + var16 = -0.14926464875852247; + } else { + if (input[8] > 779.5000000000001) { + if (input[133] > 1e-35) { + var16 = -0.036572140520852024; + } else { + if (input[183] > 1e-35) { + var16 = -0.10766853736801459; + } else { + var16 = -0.003966794968701808; + } + } + } else { + if (input[217] > 1e-35) { + if (input[5] > 5237.500000000001) { + var16 = 0.09513215942486053; + } else { + var16 = -0.03641865277445567; + } + } else { + if (input[10] > 59.50000000000001) { + var16 = 0.03177172388687933; + } else { + if (input[39] > 1e-35) { + var16 = -0.10234241303898953; + } else { + if (input[243] > 1e-35) { + var16 = -0.02966738115984321; + } else { + if (input[190] > 1e-35) { + var16 = -0.04312785336449181; + } else { + if (input[118] > 1e-35) { + var16 = -0.05808521194081524; + } else { + var16 = 0.006720381600740378; + } + } + } + } + } + } + } + } + } + } + } + } + } + let var17: number; + if (input[308] > 1e-35) { + if (input[5] > 423.00000000000006) { + if (input[133] > 1e-35) { + var17 = -0.046284053681928526; + } else { + if (input[210] > 1e-35) { + var17 = 0.000049778070699847876; + } else { + if (input[13] > 1e-35) { + var17 = 0.03328070054739309; + } else { + if (input[128] > 1e-35) { + var17 = -0.054790214922938896; + } else { + if (input[126] > 1e-35) { + var17 = -0.08524792218532945; + } else { + var17 = 0.014414055975542446; + } + } + } + } + } + } else { + if (input[1] > 38.50000000000001) { + var17 = -0.07287851335872973; + } else { + var17 = 0.005263371501687163; + } + } + } else { + if (input[9] > 7.500000000000001) { + if (input[21] > 1e-35) { + if (input[10] > 4.500000000000001) { + var17 = -0.12459748864088374; + } else { + var17 = -0.004626323021331593; + } + } else { + if (input[298] > 1e-35) { + if (input[4] > 64.50000000000001) { + var17 = 0.13044981041138526; + } else { + if (input[9] > 71.50000000000001) { + var17 = -0.056068402282406865; + } else { + if (input[9] > 12.500000000000002) { + var17 = 0.038957722962512764; + } else { + var17 = -0.04598815982492169; + } + } + } + } else { + if (input[8] > 691.5000000000001) { + if (input[126] > 1e-35) { + var17 = -0.0852126122372075; + } else { + if (input[225] > 1e-35) { + var17 = 0.10082066771689505; + } else { + if (input[1] > 161.50000000000003) { + var17 = -0.11609832500613824; + } else { + if (input[3] > 8.500000000000002) { + if (input[8] > 1685.5000000000002) { + var17 = -0.010835400874777133; + } else { + var17 = 0.004607419973807752; + } + } else { + var17 = -0.016989075258564062; + } + } + } + } + } else { + var17 = 0.009205417251698097; + } + } + } + } else { + if (input[23] > 1e-35) { + if (input[20] > 1e-35) { + var17 = 0.10184317139657878; + } else { + if (input[0] > 5724.500000000001) { + var17 = -0.1163666496650542; + } else { + if (input[1] > 106.50000000000001) { + var17 = 0.1303850608190687; + } else { + if (input[129] > 1e-35) { + var17 = 0.10745031509534769; + } else { + var17 = 0.006166901738036226; + } + } + } + } + } else { + if (input[31] > 1e-35) { + var17 = 0.010177092833155127; + } else { + if (input[13] > 1e-35) { + if (input[0] > 213.50000000000003) { + var17 = 0.005004582564506611; + } else { + var17 = -0.10481581731668346; + } + } else { + if (input[19] > 1e-35) { + var17 = -0.009850706427306281; + } else { + var17 = -0.02608226348051303; + } + } + } + } + } + } + let var18: number; + if (input[13] > 1e-35) { + if (input[1] > 64.50000000000001) { + if (input[2] > 4.500000000000001) { + var18 = -0.0024117174588695603; + } else { + var18 = -0.058339700513831916; + } + } else { + if (input[212] > 1e-35) { + if (input[0] > 2215.5000000000005) { + if (input[8] > 847.5000000000001) { + if (input[10] > 21.500000000000004) { + if (input[1] > 39.50000000000001) { + var18 = 0.04575380761203418; + } else { + var18 = -0.10025595041353463; + } + } else { + if (input[15] > 1e-35) { + var18 = 0.17705790384964004; + } else { + var18 = 0.0073813837628615014; + } + } + } else { + var18 = 0.07676373681392407; + } + } else { + var18 = -0.027167992693885996; + } + } else { + if (input[3] > 11.500000000000002) { + if (input[280] > 1e-35) { + var18 = 0.07078572910026419; + } else { + if (input[4] > 23.500000000000004) { + var18 = 0.005513918674164821; + } else { + var18 = 0.0206586476926392; + } + } + } else { + if (input[0] > 5269.500000000001) { + var18 = 0.07706773525822633; + } else { + var18 = -0.010233826953776122; + } + } + } + } + } else { + if (input[148] > 1e-35) { + if (input[8] > 1622.5000000000002) { + var18 = -0.03204783603215824; + } else { + var18 = 0.027405418223981973; + } + } else { + if (input[4] > 14.500000000000002) { + if (input[131] > 1e-35) { + if (input[9] > 1.5000000000000002) { + if (input[0] > 5026.500000000001) { + var18 = -0.0930246911392012; + } else { + var18 = 0.011173087289703683; + } + } else { + if (input[3] > 24.500000000000004) { + var18 = 0.03281421918878597; + } else { + var18 = 0.12449335091369843; + } + } + } else { + if (input[204] > 1e-35) { + var18 = 0.06634531187326123; + } else { + var18 = -0.011522999669353388; + } + } + } else { + if (input[92] > 1e-35) { + if (input[10] > 42.50000000000001) { + var18 = -0.041196758517013515; + } else { + if (input[4] > 7.500000000000001) { + var18 = -0.00002942718111029724; + } else { + if (input[4] > 6.500000000000001) { + var18 = 0.11953909558532852; + } else { + var18 = 0.03188615019450534; + } + } + } + } else { + if (input[122] > 1e-35) { + var18 = -0.0616037324662157; + } else { + if (input[101] > 1e-35) { + var18 = 0.027230889593349412; + } else { + if (input[8] > 4968.500000000001) { + var18 = -0.1113986516540856; + } else { + if (input[3] > 2.5000000000000004) { + var18 = -0.002045140426885727; + } else { + if (input[129] > 1e-35) { + var18 = 0.12641163374304432; + } else { + var18 = 0.014909826232873194; + } + } + } + } + } + } + } + } + } + let var19: number; + if (input[308] > 1e-35) { + if (input[0] > 7277.500000000001) { + var19 = -0.09337446795435; + } else { + if (input[5] > 423.00000000000006) { + if (input[133] > 1e-35) { + var19 = -0.040884836258675006; + } else { + if (input[210] > 1e-35) { + var19 = -0.0003719413278428804; + } else { + if (input[13] > 1e-35) { + var19 = 0.030287610160818174; + } else { + var19 = 0.011174130013595384; + } + } + } + } else { + if (input[1] > 38.50000000000001) { + var19 = -0.0662442170185784; + } else { + var19 = 0.004332185707008564; + } + } + } + } else { + if (input[9] > 7.500000000000001) { + if (input[145] > 1e-35) { + if (input[285] > 1e-35) { + var19 = -0.08092286307197555; + } else { + var19 = 0.029866363328584986; + } + } else { + if (input[21] > 1e-35) { + if (input[10] > 4.500000000000001) { + var19 = -0.1155211149523894; + } else { + var19 = -0.0032903546638958538; + } + } else { + if (input[149] > 1e-35) { + var19 = -0.03632198993199768; + } else { + if (input[3] > 9.500000000000002) { + if (input[8] > 999.5000000000001) { + var19 = -0.003507023626534306; + } else { + if (input[128] > 1e-35) { + if (input[4] > 13.500000000000002) { + if (input[0] > 3459.5000000000005) { + var19 = -0.025416927789760076; + } else { + var19 = 0.02777568919793122; + } + } else { + var19 = -0.10310351509769732; + } + } else { + var19 = 0.013549608903688785; + } + } + } else { + if (input[186] > 1e-35) { + var19 = 0.08513865847420551; + } else { + var19 = -0.009306721292510369; + } + } + } + } + } + } else { + if (input[31] > 1e-35) { + var19 = 0.009780833952582307; + } else { + if (input[23] > 1e-35) { + var19 = 0.011143773934157629; + } else { + if (input[210] > 1e-35) { + var19 = 0.025354797285173356; + } else { + if (input[17] > 1e-35) { + if (input[10] > 3.5000000000000004) { + var19 = -0.04846287537743046; + } else { + var19 = -0.014647271080376757; + } + } else { + if (input[2] > 5.500000000000001) { + if (input[7] > 57.50000000000001) { + var19 = -0.034224938681445764; + } else { + if (input[8] > 1641.5000000000002) { + var19 = -0.027298372075800673; + } else { + if (input[191] > 1e-35) { + if (input[10] > 18.500000000000004) { + var19 = -0.027950103994861836; + } else { + var19 = 0.14575930827829034; + } + } else { + var19 = -0.007124740389354946; + } + } + } + } else { + if (input[10] > 22.500000000000004) { + var19 = 0.013173304107866726; + } else { + var19 = -0.11119620042551365; + } + } + } + } + } + } + } + } + let var20: number; + if (input[131] > 1e-35) { + var20 = 0.01892225243240137; + } else { + if (input[308] > 1e-35) { + if (input[5] > 691.5000000000001) { + if (input[133] > 1e-35) { + var20 = -0.037118314390013646; + } else { + if (input[1] > 51.50000000000001) { + if (input[5] > 3749.5000000000005) { + if (input[8] > 58.50000000000001) { + var20 = -0.022305242912035072; + } else { + var20 = 0.024792895826340516; + } + } else { + var20 = 0.013666137278072166; + } + } else { + if (input[88] > 1e-35) { + if (input[10] > 27.500000000000004) { + var20 = 0.2080083584805785; + } else { + var20 = 0.04247197078083379; + } + } else { + if (input[10] > 40.50000000000001) { + if (input[18] > 1e-35) { + if (input[1] > 27.500000000000004) { + var20 = 0.060783227455868206; + } else { + var20 = -0.056904865557409035; + } + } else { + var20 = -0.03278952553107572; + } + } else { + if (input[192] > 1e-35) { + var20 = 0.13117402617043625; + } else { + var20 = 0.01647119888257836; + } + } + } + } + } + } else { + var20 = -0.01825870445636398; + } + } else { + if (input[9] > 6.500000000000001) { + if (input[298] > 1e-35) { + var20 = 0.026536210945939682; + } else { + if (input[8] > 691.5000000000001) { + if (input[126] > 1e-35) { + var20 = -0.07927319604548912; + } else { + if (input[10] > 3.5000000000000004) { + if (input[21] > 1e-35) { + var20 = -0.11083976837572328; + } else { + if (input[146] > 1e-35) { + var20 = -0.03359294484446772; + } else { + var20 = -0.0042815953591236475; + } + } + } else { + if (input[190] > 1e-35) { + var20 = -0.09264239592903775; + } else { + if (input[10] > 1e-35) { + var20 = 0.022282638485105657; + } else { + var20 = -0.0205994057928458; + } + } + } + } + } else { + if (input[5] > 4918.500000000001) { + var20 = 0.03430715695199153; + } else { + if (input[243] > 1e-35) { + if (input[2] > 57.50000000000001) { + var20 = 0.08935072241972036; + } else { + var20 = -0.03781647876237494; + } + } else { + var20 = 0.0062655753179671515; + } + } + } + } + } else { + if (input[31] > 1e-35) { + var20 = 0.008603500300349887; + } else { + if (input[230] > 1e-35) { + var20 = 0.03350056932774173; + } else { + if (input[23] > 1e-35) { + if (input[241] > 1e-35) { + var20 = 0.10277555508503314; + } else { + var20 = 0.0017901817172993888; + } + } else { + if (input[2] > 98.50000000000001) { + var20 = -0.05920081229672715; + } else { + var20 = -0.015722173275739208; + } + } + } + } + } + } + } + let var21: number; + if (input[13] > 1e-35) { + if (input[118] > 1e-35) { + var21 = 0.07957905150112207; + } else { + if (input[1] > 125.50000000000001) { + var21 = -0.0662620579858685; + } else { + if (input[145] > 1e-35) { + var21 = 0.029682040828779843; + } else { + if (input[19] > 1e-35) { + if (input[6] > 15.500000000000002) { + var21 = -0.0009597832580977798; + } else { + var21 = -0.081474760755753; + } + } else { + if (input[212] > 1e-35) { + var21 = 0.03637001492325179; + } else { + var21 = 0.006912305498963309; + } + } + } + } + } + } else { + if (input[32] > 1e-35) { + var21 = -0.03919900630910754; + } else { + if (input[134] > 1e-35) { + var21 = -0.036225295529777886; + } else { + if (input[4] > 4.500000000000001) { + if (input[5] > 384.50000000000006) { + if (input[204] > 1e-35) { + var21 = 0.06671440854602108; + } else { + if (input[136] > 1e-35) { + var21 = -0.07577364230133474; + } else { + if (input[148] > 1e-35) { + if (input[4] > 7.500000000000001) { + var21 = 0.026430947016830915; + } else { + var21 = -0.04075501264495112; + } + } else { + if (input[9] > 93.50000000000001) { + var21 = -0.04353169430417609; + } else { + if (input[50] > 1e-35) { + var21 = -0.1411224537622882; + } else { + if (input[17] > 1e-35) { + if (input[49] > 1e-35) { + var21 = 0.068392679163672; + } else { + if (input[10] > 1.5000000000000002) { + var21 = -0.0209659792007492; + } else { + var21 = -0.0004393235559249831; + } + } + } else { + if (input[133] > 1e-35) { + if (input[9] > 64.50000000000001) { + var21 = 0.07254524592323175; + } else { + var21 = -0.0319087835282534; + } + } else { + var21 = 0.00037444813327793425; + } + } + } + } + } + } + } + } else { + var21 = -0.025138768151370408; + } + } else { + if (input[243] > 1e-35) { + var21 = -0.050010891710502096; + } else { + if (input[94] > 1e-35) { + var21 = -0.0817513550778599; + } else { + if (input[122] > 1e-35) { + var21 = -0.061038875809822285; + } else { + if (input[19] > 1e-35) { + if (input[8] > 1085.5000000000002) { + var21 = -0.008408408775061623; + } else { + if (input[2] > 5.500000000000001) { + if (input[218] > 1e-35) { + var21 = 0.1454877641381946; + } else { + var21 = 0.053787998331240316; + } + } else { + if (input[9] > 33.50000000000001) { + var21 = 0.08602629796680285; + } else { + var21 = -0.03895127455803038; + } + } + } + } else { + var21 = 0.008830878042315722; + } + } + } + } + } + } + } + } + let var22: number; + if (input[131] > 1e-35) { + var22 = 0.01687979707990516; + } else { + if (input[8] > 2915.5000000000005) { + if (input[297] > 1e-35) { + var22 = 0.07473600489975568; + } else { + if (input[0] > 93.50000000000001) { + var22 = -0.021596848506011502; + } else { + var22 = -0.13840802327735696; + } + } + } else { + if (input[230] > 1e-35) { + if (input[4] > 6.500000000000001) { + if (input[0] > 4977.500000000001) { + var22 = 0.10264284346448256; + } else { + var22 = 0.031042487183181262; + } + } else { + var22 = -0.016653982936827776; + } + } else { + if (input[4] > 60.50000000000001) { + if (input[10] > 75.50000000000001) { + var22 = 0.04226403420647408; + } else { + if (input[10] > 1e-35) { + if (input[0] > 4733.500000000001) { + var22 = 0.006271403149804702; + } else { + var22 = -0.030013637555715046; + } + } else { + if (input[0] > 4449.500000000001) { + var22 = -0.06556876058654929; + } else { + var22 = 0.06437994816903034; + } + } + } + } else { + if (input[32] > 1e-35) { + var22 = -0.043814577251655815; + } else { + if (input[308] > 1e-35) { + if (input[0] > 7277.500000000001) { + var22 = -0.09349726304052086; + } else { + if (input[210] > 1e-35) { + var22 = -0.0035960132209098003; + } else { + if (input[5] > 691.5000000000001) { + if (input[133] > 1e-35) { + var22 = -0.029188394315052574; + } else { + var22 = 0.017219308333820193; + } + } else { + var22 = -0.017378928852189585; + } + } + } + } else { + if (input[9] > 6.500000000000001) { + if (input[0] > 2653.5000000000005) { + if (input[149] > 1e-35) { + var22 = -0.04428555753857688; + } else { + var22 = 0.0001456106867817353; + } + } else { + if (input[5] > 213.50000000000003) { + var22 = 0.01740292726636365; + } else { + var22 = -0.011361718115556464; + } + } + } else { + if (input[7] > 4.500000000000001) { + if (input[0] > 316.50000000000006) { + if (input[19] > 1e-35) { + if (input[10] > 54.50000000000001) { + var22 = 0.03410288911259329; + } else { + if (input[121] > 1e-35) { + var22 = -0.06056527462120627; + } else { + if (input[8] > 2592.5000000000005) { + var22 = 0.12166808844363577; + } else { + if (input[191] > 1e-35) { + var22 = 0.11669879218998758; + } else { + var22 = -0.001664858391716235; + } + } + } + } + } else { + var22 = -0.01262927450503166; + } + } else { + var22 = -0.04506589951879664; + } + } else { + if (input[227] > 1e-35) { + var22 = -0.08548904959752329; + } else { + var22 = 0.02156080776537726; + } + } + } + } + } + } + } + } + } + let var23: number; + if (input[306] > 1e-35) { + if (input[149] > 1e-35) { + var23 = -0.1389218965136736; + } else { + var23 = -0.032218642644416894; + } + } else { + if (input[13] > 1e-35) { + var23 = 0.006465035217331847; + } else { + if (input[50] > 1e-35) { + var23 = -0.1381687930130022; + } else { + if (input[179] > 1e-35) { + var23 = -0.13112784985951215; + } else { + if (input[148] > 1e-35) { + if (input[8] > 1726.5000000000002) { + var23 = -0.03262719498763048; + } else { + var23 = 0.023342916702125613; + } + } else { + if (input[191] > 1e-35) { + var23 = 0.030005484947580197; + } else { + if (input[4] > 4.500000000000001) { + if (input[204] > 1e-35) { + var23 = 0.047767773119269434; + } else { + if (input[136] > 1e-35) { + if (input[0] > 1937.5000000000002) { + var23 = -0.09989343595668776; + } else { + var23 = 0.06533942033334243; + } + } else { + if (input[15] > 1e-35) { + if (input[9] > 86.50000000000001) { + var23 = -0.10577989354150097; + } else { + if (input[8] > 668.5000000000001) { + if (input[126] > 1e-35) { + var23 = -0.09165257825246746; + } else { + if (input[9] > 32.50000000000001) { + var23 = 0.02484870392366004; + } else { + var23 = -0.008499493096971395; + } + } + } else { + if (input[8] > 24.500000000000004) { + var23 = 0.02459679192828244; + } else { + var23 = -0.010527978013140512; + } + } + } + } else { + if (input[25] > 1e-35) { + if (input[217] > 1e-35) { + var23 = 0.0015644546318714849; + } else { + var23 = -0.06579524865022705; + } + } else { + var23 = -0.0060233890975120614; + } + } + } + } + } else { + if (input[122] > 1e-35) { + if (input[1] > 36.50000000000001) { + var23 = 0.03331853632960164; + } else { + var23 = -0.09482264761126993; + } + } else { + if (input[19] > 1e-35) { + if (input[8] > 1430.5000000000002) { + var23 = -0.019091477207111116; + } else { + var23 = 0.037878468575478504; + } + } else { + if (input[94] > 1e-35) { + var23 = -0.08013082284576584; + } else { + if (input[4] > 2.5000000000000004) { + if (input[186] > 1e-35) { + var23 = 0.16919658785098224; + } else { + if (input[243] > 1e-35) { + var23 = -0.06580584936754524; + } else { + var23 = 0.01567555159935563; + } + } + } else { + if (input[129] > 1e-35) { + var23 = 0.06721746994993226; + } else { + if (input[10] > 32.50000000000001) { + var23 = -0.046394462507797975; + } else { + var23 = -0.006436180519584767; + } + } + } + } + } + } + } + } + } + } + } + } + } + let var24: number; + if (input[131] > 1e-35) { + var24 = 0.015039096856208693; + } else { + if (input[8] > 779.5000000000001) { + if (input[145] > 1e-35) { + var24 = 0.019122095523977856; + } else { + if (input[298] > 1e-35) { + var24 = 0.023828936462317443; + } else { + if (input[1] > 23.500000000000004) { + if (input[5] > 384.50000000000006) { + if (input[7] > 59.50000000000001) { + var24 = -0.026094309429557913; + } else { + if (input[204] > 1e-35) { + var24 = 0.09163404305658318; + } else { + if (input[1] > 27.500000000000004) { + if (input[149] > 1e-35) { + if (input[6] > 34.50000000000001) { + var24 = 0.012643810980689466; + } else { + var24 = -0.07884161741497837; + } + } else { + var24 = -0.0025267379810891104; + } + } else { + if (input[2] > 43.50000000000001) { + if (input[0] > 2860.5000000000005) { + var24 = 0.04493082949897325; + } else { + var24 = 0.18046359750455776; + } + } else { + if (input[7] > 18.500000000000004) { + var24 = -0.018667348656891496; + } else { + var24 = 0.02584325784698236; + } + } + } + } + } + } else { + var24 = -0.045696524897545915; + } + } else { + if (input[0] > 3321.5000000000005) { + if (input[201] > 1e-35) { + var24 = 0.04749240016989375; + } else { + var24 = -0.0333334578246718; + } + } else { + if (input[5] > 3276.5000000000005) { + var24 = 0.11330554740098908; + } else { + if (input[7] > 94.50000000000001) { + var24 = 0.1296600395033268; + } else { + var24 = -0.003576436308940934; + } + } + } + } + } + } + } else { + if (input[15] > 1e-35) { + if (input[183] > 1e-35) { + var24 = -0.13787130789142835; + } else { + if (input[0] > 1847.5000000000002) { + var24 = 0.017915229729920556; + } else { + if (input[10] > 23.500000000000004) { + if (input[10] > 31.500000000000004) { + if (input[6] > 7.500000000000001) { + var24 = 0.028856848462727104; + } else { + var24 = -0.11197632885851168; + } + } else { + var24 = 0.08169801342016791; + } + } else { + if (input[1] > 22.500000000000004) { + var24 = -0.021052888644970163; + } else { + var24 = 0.019048604298876753; + } + } + } + } + } else { + if (input[7] > 4.500000000000001) { + var24 = -0.002603328695276418; + } else { + if (input[7] > 1.5000000000000002) { + if (input[2] > 5.500000000000001) { + var24 = 0.03432638833359197; + } else { + var24 = -0.0036767863082454973; + } + } else { + if (input[1] > 48.50000000000001) { + var24 = 0.03087375270128195; + } else { + if (input[2] > 3.5000000000000004) { + var24 = -0.04219917149740248; + } else { + var24 = 0.018818493993207935; + } + } + } + } + } + } + } + let var25: number; + if (input[306] > 1e-35) { + var25 = -0.04076858123502297; + } else { + if (input[13] > 1e-35) { + if (input[1] > 67.50000000000001) { + if (input[9] > 14.500000000000002) { + if (input[9] > 53.50000000000001) { + if (input[8] > 1971.5000000000002) { + var25 = -0.09091897542577475; + } else { + var25 = 0.04042943082645558; + } + } else { + if (input[218] > 1e-35) { + var25 = 0.056254985867151; + } else { + var25 = -0.053848117950183044; + } + } + } else { + var25 = 0.003881630017086845; + } + } else { + if (input[5] > 5152.500000000001) { + if (input[8] > 857.5000000000001) { + if (input[6] > 28.500000000000004) { + var25 = 0.021581808008986944; + } else { + var25 = -0.05639286496176611; + } + } else { + var25 = 0.052838875036198954; + } + } else { + if (input[5] > 50.50000000000001) { + if (input[5] > 4082.5000000000005) { + if (input[17] > 1e-35) { + var25 = 0.023061479860228728; + } else { + if (input[145] > 1e-35) { + if (input[9] > 10.500000000000002) { + var25 = 0.023885302967553288; + } else { + var25 = 0.1617794086125622; + } + } else { + if (input[212] > 1e-35) { + var25 = 0.04504545345658806; + } else { + if (input[3] > 17.500000000000004) { + if (input[4] > 45.50000000000001) { + var25 = -0.03948072448245435; + } else { + if (input[1] > 47.50000000000001) { + if (input[9] > 18.500000000000004) { + var25 = 0.01894935813286188; + } else { + var25 = -0.06449356357429188; + } + } else { + var25 = 0.012297239104320094; + } + } + } else { + if (input[1] > 26.500000000000004) { + if (input[8] > 33.50000000000001) { + var25 = -0.034718828212885515; + } else { + var25 = 0.0898976288814321; + } + } else { + if (input[1] > 17.500000000000004) { + var25 = -0.15440137451988326; + } else { + var25 = -0.03864183216821465; + } + } + } + } + } + } + } else { + var25 = 0.009988507307006308; + } + } else { + var25 = -0.08540311947043305; + } + } + } + } else { + if (input[50] > 1e-35) { + var25 = -0.13323659732101975; + } else { + if (input[134] > 1e-35) { + var25 = -0.031820386486894385; + } else { + if (input[32] > 1e-35) { + if (input[8] > 2302.5000000000005) { + var25 = 0.08082476177379844; + } else { + var25 = -0.041665761903645876; + } + } else { + if (input[179] > 1e-35) { + var25 = -0.12405023987936657; + } else { + if (input[39] > 1e-35) { + var25 = -0.06247416524997478; + } else { + if (input[138] > 1e-35) { + var25 = -0.10724031753676487; + } else { + var25 = -0.0005423122305122404; + } + } + } + } + } + } + } + } + let var26: number; + if (input[308] > 1e-35) { + var26 = 0.006160742906729798; + } else { + if (input[190] > 1e-35) { + if (input[0] > 2461.5000000000005) { + if (input[10] > 22.500000000000004) { + var26 = 0.023223358334607133; + } else { + var26 = -0.04383410185346742; + } + } else { + var26 = -0.08542395045055405; + } + } else { + if (input[297] > 1e-35) { + if (input[8] > 51.50000000000001) { + if (input[1] > 13.500000000000002) { + var26 = 0.023406489302867494; + } else { + var26 = -0.085521220804058; + } + } else { + var26 = -0.02921899554854833; + } + } else { + if (input[298] > 1e-35) { + if (input[9] > 12.500000000000002) { + var26 = 0.028120059780969632; + } else { + var26 = -0.04211009474298743; + } + } else { + if (input[294] > 1e-35) { + var26 = -0.05040415676618239; + } else { + if (input[86] > 1e-35) { + if (input[1] > 36.50000000000001) { + var26 = -0.0993035220737934; + } else { + var26 = -0.0005384930611060366; + } + } else { + if (input[230] > 1e-35) { + if (input[4] > 6.500000000000001) { + var26 = 0.029770210551187937; + } else { + var26 = -0.016272917551655715; + } + } else { + if (input[4] > 60.50000000000001) { + if (input[280] > 1e-35) { + var26 = 0.06421359317599738; + } else { + var26 = -0.01963732469244167; + } + } else { + if (input[218] > 1e-35) { + if (input[3] > 3.5000000000000004) { + var26 = 0.024368404612215164; + } else { + var26 = -0.04045232374803373; + } + } else { + if (input[131] > 1e-35) { + var26 = 0.017372701982485795; + } else { + if (input[120] > 1e-35) { + var26 = 0.08812710275150198; + } else { + if (input[18] > 1e-35) { + if (input[90] > 1e-35) { + var26 = 0.18451364351180236; + } else { + if (input[7] > 33.50000000000001) { + var26 = -0.03850813130183531; + } else { + if (input[195] > 1e-35) { + var26 = 0.06966114053446336; + } else { + if (input[3] > 16.500000000000004) { + var26 = -0.0012869181693341211; + } else { + if (input[0] > 4242.500000000001) { + var26 = -0.054625548611291035; + } else { + var26 = -0.014431095117473881; + } + } + } + } + } + } else { + if (input[5] > 4558.500000000001) { + if (input[8] > 1.5000000000000002) { + var26 = 0.006302103427145562; + } else { + var26 = 0.13967622319898698; + } + } else { + if (input[121] > 1e-35) { + var26 = -0.038798585213145644; + } else { + if (input[5] > 4544.500000000001) { + var26 = -0.08050498033009466; + } else { + var26 = -0.002986974112681435; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + let var27: number; + if (input[0] > 384.50000000000006) { + if (input[2] > 101.50000000000001) { + if (input[1] > 16.500000000000004) { + var27 = -0.03461119351456781; + } else { + var27 = 0.05659026566680352; + } + } else { + if (input[306] > 1e-35) { + if (input[2] > 14.500000000000002) { + if (input[149] > 1e-35) { + var27 = -0.12404435523286539; + } else { + var27 = -0.0034376913880382956; + } + } else { + var27 = -0.09821622245095822; + } + } else { + if (input[131] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var27 = 0.0037507103585310234; + } else { + var27 = 0.03610387965829944; + } + } else { + if (input[8] > 999.5000000000001) { + if (input[9] > 137.50000000000003) { + var27 = -0.11985021663179699; + } else { + if (input[0] > 1847.5000000000002) { + if (input[126] > 1e-35) { + var27 = -0.04832024079663151; + } else { + if (input[37] > 1e-35) { + var27 = -0.037103393468366934; + } else { + var27 = -0.004248086592531705; + } + } + } else { + if (input[8] > 3084.0000000000005) { + if (input[9] > 43.50000000000001) { + var27 = 0.032539071163832034; + } else { + if (input[5] > 1643.5000000000002) { + var27 = 0.036408625378035665; + } else { + if (input[0] > 1500.5000000000002) { + var27 = -0.1346358322854993; + } else { + var27 = -0.027586559522081014; + } + } + } + } else { + if (input[3] > 1e-35) { + if (input[190] > 1e-35) { + var27 = -0.1133991164577881; + } else { + if (input[9] > 52.50000000000001) { + var27 = -0.024478640359723122; + } else { + var27 = 0.03673777861098756; + } + } + } else { + var27 = -0.1037451237591819; + } + } + } + } + } else { + if (input[230] > 1e-35) { + if (input[9] > 48.50000000000001) { + if (input[10] > 20.500000000000004) { + var27 = 0.002583438691776944; + } else { + var27 = 0.10773520810108106; + } + } else { + if (input[9] > 12.500000000000002) { + if (input[1] > 16.500000000000004) { + var27 = -0.02141222346712401; + } else { + var27 = 0.06392462314316179; + } + } else { + if (input[4] > 12.500000000000002) { + var27 = 0.08700122294434816; + } else { + if (input[8] > 267.50000000000006) { + var27 = 0.056923170082743224; + } else { + var27 = -0.07716309825583327; + } + } + } + } + } else { + if (input[32] > 1e-35) { + var27 = -0.03961343943752142; + } else { + var27 = 0.002674914122888783; + } + } + } + } + } + } + } else { + if (input[1] > 42.50000000000001) { + var27 = -0.05217539654421676; + } else { + if (input[145] > 1e-35) { + var27 = 0.09553630282946368; + } else { + var27 = -0.009424791262477729; + } + } + } + let var28: number; + if (input[183] > 1e-35) { + var28 = -0.05753337139158443; + } else { + if (input[308] > 1e-35) { + var28 = 0.00562436671450989; + } else { + if (input[9] > 7.500000000000001) { + if (input[21] > 1e-35) { + if (input[10] > 8.500000000000002) { + var28 = -0.10477869875380448; + } else { + var28 = -0.0070301869937306055; + } + } else { + if (input[3] > 9.500000000000002) { + if (input[8] > 1765.5000000000002) { + if (input[0] > 4571.500000000001) { + var28 = -0.12526505173232894; + } else { + if (input[10] > 1e-35) { + if (input[9] > 71.50000000000001) { + var28 = -0.04442302951713574; + } else { + var28 = 0.00012409888451734224; + } + } else { + var28 = -0.092199119633697; + } + } + } else { + if (input[225] > 1e-35) { + var28 = 0.13773072450201831; + } else { + if (input[0] > 2882.5000000000005) { + var28 = 0.0028540012229920533; + } else { + if (input[298] > 1e-35) { + var28 = 0.07134486044361629; + } else { + var28 = 0.014297412329837425; + } + } + } + } + } else { + if (input[145] > 1e-35) { + var28 = 0.05608385321902638; + } else { + if (input[92] > 1e-35) { + var28 = 0.038298413603926135; + } else { + if (input[107] > 1e-35) { + if (input[2] > 6.500000000000001) { + var28 = -0.0039957800609801315; + } else { + var28 = 0.0776927564241081; + } + } else { + if (input[203] > 1e-35) { + var28 = -0.05502900859432093; + } else { + if (input[105] > 1e-35) { + var28 = 0.06062892720841595; + } else { + var28 = -0.009574839629252128; + } + } + } + } + } + } + } + } else { + if (input[31] > 1e-35) { + var28 = 0.009488858841144216; + } else { + if (input[23] > 1e-35) { + if (input[20] > 1e-35) { + var28 = 0.08818126313644752; + } else { + if (input[8] > 161.50000000000003) { + var28 = 0.014353968957885408; + } else { + var28 = -0.022240738532827903; + } + } + } else { + if (input[210] > 1e-35) { + var28 = 0.024648862719806694; + } else { + if (input[2] > 5.500000000000001) { + if (input[4] > 4.500000000000001) { + if (input[17] > 1e-35) { + if (input[10] > 16.500000000000004) { + var28 = -0.043902062079383485; + } else { + var28 = -0.014741559220396223; + } + } else { + var28 = -0.00934935734853194; + } + } else { + if (input[6] > 32.50000000000001) { + var28 = 0.1514593126307404; + } else { + var28 = 0.010771222510801532; + } + } + } else { + if (input[10] > 22.500000000000004) { + var28 = 0.01412495209334078; + } else { + var28 = -0.08576940379502533; + } + } + } + } + } + } + } + } + let var29: number; + if (input[0] > 384.50000000000006) { + if (input[84] > 1e-35) { + var29 = -0.06647690967306838; + } else { + if (input[2] > 101.50000000000001) { + var29 = -0.024451334501552457; + } else { + if (input[306] > 1e-35) { + var29 = -0.034517188927733505; + } else { + if (input[131] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var29 = 0.0031858381443673127; + } else { + var29 = 0.032574927024450646; + } + } else { + if (input[204] > 1e-35) { + if (input[1] > 62.50000000000001) { + var29 = -0.08601340441214533; + } else { + if (input[1] > 29.500000000000004) { + var29 = 0.10487598629539963; + } else { + if (input[8] > 597.5000000000001) { + var29 = -0.0786529133673238; + } else { + var29 = 0.08689436600511559; + } + } + } + } else { + if (input[8] > 779.5000000000001) { + if (input[10] > 2.5000000000000004) { + if (input[9] > 100.50000000000001) { + var29 = -0.04883600353740688; + } else { + if (input[126] > 1e-35) { + var29 = -0.03794042763348827; + } else { + var29 = -0.003358871967539988; + } + } + } else { + if (input[210] > 1e-35) { + var29 = 0.054991356498447566; + } else { + if (input[6] > 19.500000000000004) { + var29 = -0.007418396981635549; + } else { + var29 = 0.018032606049498613; + } + } + } + } else { + if (input[18] > 1e-35) { + if (input[7] > 35.50000000000001) { + if (input[2] > 44.50000000000001) { + var29 = -0.02143003429501711; + } else { + var29 = -0.09016000554055564; + } + } else { + if (input[1] > 19.500000000000004) { + if (input[1] > 42.50000000000001) { + if (input[8] > 17.500000000000004) { + var29 = -0.006636355416244082; + } else { + var29 = -0.06483095743431454; + } + } else { + if (input[4] > 21.500000000000004) { + var29 = -0.028975965946833545; + } else { + var29 = 0.022012264796522657; + } + } + } else { + var29 = -0.06653648243193663; + } + } + } else { + if (input[5] > 4593.500000000001) { + var29 = 0.01753551428088607; + } else { + if (input[217] > 1e-35) { + var29 = -0.028864824937700297; + } else { + if (input[94] > 1e-35) { + var29 = -0.04885192273020658; + } else { + if (input[279] > 1e-35) { + var29 = 0.08105715462329498; + } else { + if (input[121] > 1e-35) { + var29 = -0.04576676034750651; + } else { + var29 = 0.004795141324949362; + } + } + } + } + } + } + } + } + } + } + } + } + } else { + if (input[1] > 42.50000000000001) { + var29 = -0.047446619702809195; + } else { + if (input[145] > 1e-35) { + var29 = 0.08400495571952321; + } else { + var29 = -0.00854528836489364; + } + } + } + let var30: number; + if (input[294] > 1e-35) { + var30 = -0.042529778074638265; + } else { + if (input[266] > 1e-35) { + var30 = -0.1180276669679798; + } else { + if (input[134] > 1e-35) { + var30 = -0.026818144353279623; + } else { + if (input[183] > 1e-35) { + var30 = -0.05120747503479363; + } else { + if (input[227] > 1e-35) { + if (input[8] > 1641.5000000000002) { + var30 = -0.07265906898294434; + } else { + if (input[4] > 12.500000000000002) { + if (input[17] > 1e-35) { + var30 = -0.027516137530797014; + } else { + if (input[0] > 4331.500000000001) { + if (input[1] > 64.50000000000001) { + var30 = -0.03049646619610203; + } else { + if (input[1] > 50.50000000000001) { + var30 = 0.20634590755061122; + } else { + var30 = 0.06956378103625731; + } + } + } else { + if (input[0] > 3770.5000000000005) { + var30 = -0.07946414366134913; + } else { + if (input[19] > 1e-35) { + var30 = 0.17083312065604694; + } else { + if (input[2] > 21.500000000000004) { + var30 = -0.02327981978127724; + } else { + var30 = 0.129717297518715; + } + } + } + } + } + } else { + if (input[145] > 1e-35) { + var30 = 0.006891245076133524; + } else { + var30 = -0.0789123467863741; + } + } + } + } else { + if (input[3] > 99.50000000000001) { + var30 = -0.02022281202803071; + } else { + if (input[302] > 1e-35) { + if (input[10] > 47.50000000000001) { + var30 = 0.06447639919732716; + } else { + var30 = -0.05457561977645972; + } + } else { + if (input[306] > 1e-35) { + var30 = -0.029995903305383882; + } else { + if (input[191] > 1e-35) { + var30 = 0.030596508110850414; + } else { + if (input[242] > 1e-35) { + var30 = -0.024085578702020216; + } else { + if (input[8] > 3198.5000000000005) { + if (input[297] > 1e-35) { + var30 = 0.09518584795377832; + } else { + var30 = -0.018197744600833596; + } + } else { + if (input[13] > 1e-35) { + var30 = 0.006751790086127549; + } else { + if (input[148] > 1e-35) { + var30 = 0.01904174573618417; + } else { + if (input[99] > 1e-35) { + var30 = 0.025287735102561926; + } else { + if (input[4] > 14.500000000000002) { + var30 = -0.004364337681643273; + } else { + if (input[1] > 15.500000000000002) { + if (input[35] > 1e-35) { + var30 = -0.09467943982430241; + } else { + if (input[243] > 1e-35) { + var30 = -0.02521824751996268; + } else { + var30 = 0.005437570718352172; + } + } + } else { + var30 = -0.022476214821960674; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + let var31: number; + if (input[0] > 384.50000000000006) { + if (input[84] > 1e-35) { + var31 = -0.06088131453064195; + } else { + if (input[147] > 1e-35) { + var31 = -0.05332792965930566; + } else { + if (input[135] > 1e-35) { + if (input[9] > 32.50000000000001) { + var31 = 0.04219361472548491; + } else { + var31 = -0.07227529211725771; + } + } else { + if (input[10] > 4.500000000000001) { + if (input[21] > 1e-35) { + var31 = -0.0787279848043689; + } else { + if (input[17] > 1e-35) { + if (input[3] > 18.500000000000004) { + if (input[188] > 1e-35) { + var31 = -0.054347604504400286; + } else { + if (input[0] > 3544.5000000000005) { + if (input[0] > 5850.500000000001) { + var31 = -0.11431764534511478; + } else { + var31 = 0.013549717238356157; + } + } else { + var31 = -0.020987333767091276; + } + } + } else { + if (input[6] > 2.5000000000000004) { + var31 = -0.02914877855133127; + } else { + var31 = 0.08483464900160231; + } + } + } else { + if (input[8] > 58.50000000000001) { + if (input[183] > 1e-35) { + var31 = -0.10087072787978416; + } else { + if (input[37] > 1e-35) { + var31 = -0.030467397753331196; + } else { + if (input[229] > 1e-35) { + var31 = -0.1017559811057469; + } else { + if (input[4] > 20.500000000000004) { + var31 = -0.00413177742240167; + } else { + if (input[20] > 1e-35) { + var31 = 0.05213315982685969; + } else { + var31 = 0.0037921635866823133; + } + } + } + } + } + } else { + if (input[8] > 51.50000000000001) { + var31 = 0.07327913092421544; + } else { + if (input[6] > 49.50000000000001) { + var31 = -0.03457694284156811; + } else { + if (input[6] > 18.500000000000004) { + if (input[7] > 17.500000000000004) { + var31 = 0.02744420891894289; + } else { + var31 = 0.11288946357194463; + } + } else { + var31 = 0.003482908820966248; + } + } + } + } + } + } + } else { + if (input[18] > 1e-35) { + if (input[1] > 20.500000000000004) { + if (input[7] > 4.500000000000001) { + var31 = -0.012329314369909049; + } else { + var31 = 0.026816658655600168; + } + } else { + var31 = -0.0872405354618811; + } + } else { + var31 = 0.007872673500247845; + } + } + } + } + } + } else { + if (input[1] > 42.50000000000001) { + var31 = -0.04309044198258254; + } else { + if (input[145] > 1e-35) { + var31 = 0.07572529147860785; + } else { + if (input[7] > 5.500000000000001) { + var31 = -0.013837187093264945; + } else { + if (input[1] > 17.500000000000004) { + var31 = 0.04208698439539668; + } else { + var31 = -0.06284346769019863; + } + } + } + } + } + let var32: number; + if (input[294] > 1e-35) { + var32 = -0.0384794324818203; + } else { + if (input[266] > 1e-35) { + var32 = -0.1087205883821061; + } else { + if (input[32] > 1e-35) { + if (input[8] > 2302.5000000000005) { + var32 = 0.07432960094940501; + } else { + var32 = -0.035248735855751855; + } + } else { + if (input[134] > 1e-35) { + var32 = -0.02456191365284949; + } else { + if (input[121] > 1e-35) { + if (input[0] > 4720.500000000001) { + if (input[1] > 39.50000000000001) { + var32 = -0.01706896375068821; + } else { + var32 = 0.08212247914968074; + } + } else { + if (input[2] > 59.50000000000001) { + var32 = -0.09546478958824225; + } else { + if (input[6] > 53.50000000000001) { + var32 = 0.12317082897575611; + } else { + if (input[1] > 56.50000000000001) { + if (input[4] > 7.500000000000001) { + if (input[0] > 3560.5000000000005) { + var32 = 0.02816463285971267; + } else { + var32 = 0.15449139016588445; + } + } else { + var32 = -0.10199787406123524; + } + } else { + var32 = -0.038068684323297096; + } + } + } + } + } else { + if (input[223] > 1e-35) { + if (input[8] > 668.5000000000001) { + var32 = -0.13924786681478077; + } else { + var32 = -0.0072772442570213335; + } + } else { + if (input[39] > 1e-35) { + var32 = -0.05392786531177836; + } else { + if (input[0] > 93.50000000000001) { + if (input[40] > 1e-35) { + var32 = -0.054059371343144036; + } else { + if (input[306] > 1e-35) { + if (input[2] > 14.500000000000002) { + if (input[149] > 1e-35) { + var32 = -0.11174465335620831; + } else { + var32 = 0.00013144040097180107; + } + } else { + var32 = -0.08493919336681105; + } + } else { + if (input[42] > 1e-35) { + var32 = -0.11078582572836196; + } else { + if (input[84] > 1e-35) { + if (input[4] > 17.500000000000004) { + var32 = -0.015540659878839153; + } else { + var32 = -0.14442609417300142; + } + } else { + if (input[21] > 1e-35) { + var32 = -0.025251979447574083; + } else { + var32 = 0.0023698372645272847; + } + } + } + } + } + } else { + if (input[18] > 1e-35) { + var32 = 0.07269739695712212; + } else { + if (input[8] > 2592.5000000000005) { + var32 = -0.1460388776448558; + } else { + if (input[9] > 30.500000000000004) { + if (input[1] > 23.500000000000004) { + var32 = -0.01835130329646532; + } else { + if (input[9] > 45.50000000000001) { + var32 = 0.02023047454629885; + } else { + var32 = 0.16469378262221102; + } + } + } else { + var32 = -0.042975030085836426; + } + } + } + } + } + } + } + } + } + } + } + let var33: number; + if (input[8] > 2915.5000000000005) { + if (input[297] > 1e-35) { + var33 = 0.06257393915394144; + } else { + if (input[0] > 93.50000000000001) { + if (input[4] > 1.5000000000000002) { + var33 = -0.01034964686484714; + } else { + var33 = -0.07357437440667927; + } + } else { + var33 = -0.11987794734779106; + } + } + } else { + if (input[298] > 1e-35) { + if (input[8] > 81.50000000000001) { + if (input[0] > 3370.5000000000005) { + if (input[8] > 155.50000000000003) { + if (input[8] > 660.5000000000001) { + if (input[8] > 2134.5000000000005) { + var33 = -0.09476398869062203; + } else { + if (input[9] > 72.50000000000001) { + var33 = -0.0757383854264379; + } else { + var33 = 0.02806542779508718; + } + } + } else { + var33 = -0.05147742568418084; + } + } else { + var33 = 0.10212721564444344; + } + } else { + var33 = 0.0518263760642861; + } + } else { + var33 = -0.08743405377022222; + } + } else { + if (input[189] > 1e-35) { + if (input[0] > 5269.500000000001) { + var33 = -0.10669213185972036; + } else { + var33 = 0.027050434286384796; + } + } else { + if (input[302] > 1e-35) { + var33 = -0.0407832394672723; + } else { + if (input[116] > 1e-35) { + if (input[10] > 38.50000000000001) { + var33 = 0.06354599160071946; + } else { + if (input[1] > 67.50000000000001) { + var33 = 0.05317447949011187; + } else { + var33 = -0.059138165935307165; + } + } + } else { + if (input[212] > 1e-35) { + if (input[19] > 1e-35) { + var33 = -0.09369289448773599; + } else { + if (input[0] > 2215.5000000000005) { + var33 = 0.04077965380363924; + } else { + if (input[0] > 807.5000000000001) { + var33 = -0.0591771776458298; + } else { + var33 = 0.057315736906679376; + } + } + } + } else { + if (input[308] > 1e-35) { + if (input[1] > 52.50000000000001) { + if (input[5] > 3749.5000000000005) { + var33 = -0.016323380219241672; + } else { + var33 = 0.007291062979527741; + } + } else { + if (input[210] > 1e-35) { + if (input[8] > 1641.5000000000002) { + var33 = 0.03720704290087811; + } else { + var33 = -0.008730548158766654; + } + } else { + if (input[4] > 80.50000000000001) { + var33 = -0.05346644687473197; + } else { + var33 = 0.014596824736762107; + } + } + } + } else { + if (input[218] > 1e-35) { + if (input[3] > 3.5000000000000004) { + var33 = 0.019984510398089086; + } else { + var33 = -0.03917825025861855; + } + } else { + if (input[9] > 170.50000000000003) { + var33 = -0.09759719821334525; + } else { + var33 = -0.0023586682752856298; + } + } + } + } + } + } + } + } + } + let var34: number; + if (input[183] > 1e-35) { + if (input[17] > 1e-35) { + var34 = 0.030100940443356424; + } else { + if (input[10] > 1.5000000000000002) { + var34 = -0.10861112216742408; + } else { + var34 = 0.017680668976453255; + } + } + } else { + if (input[227] > 1e-35) { + if (input[17] > 1e-35) { + if (input[2] > 16.500000000000004) { + var34 = -0.032062878390325456; + } else { + var34 = -0.10808232631806887; + } + } else { + if (input[8] > 1641.5000000000002) { + var34 = -0.06147013392655731; + } else { + if (input[4] > 12.500000000000002) { + var34 = 0.03324767551088266; + } else { + if (input[145] > 1e-35) { + var34 = 0.028851633810612017; + } else { + var34 = -0.054871239091792784; + } + } + } + } + } else { + if (input[134] > 1e-35) { + var34 = -0.023813968121342108; + } else { + if (input[266] > 1e-35) { + var34 = -0.10037039667146351; + } else { + if (input[222] > 1e-35) { + if (input[0] > 612.5000000000001) { + if (input[10] > 1e-35) { + if (input[8] > 1939.5000000000002) { + var34 = -0.055566877553100726; + } else { + if (input[2] > 24.500000000000004) { + if (input[8] > 182.50000000000003) { + if (input[10] > 43.50000000000001) { + if (input[10] > 55.50000000000001) { + var34 = -0.025350325484720576; + } else { + var34 = 0.1579024598549572; + } + } else { + if (input[9] > 2.5000000000000004) { + if (input[0] > 3746.5000000000005) { + var34 = 0.056817276537534815; + } else { + var34 = -0.07674158463557636; + } + } else { + var34 = -0.06335553143454145; + } + } + } else { + if (input[1] > 56.50000000000001) { + var34 = 0.16390494217299284; + } else { + var34 = -0.0027330160430847177; + } + } + } else { + if (input[10] > 36.50000000000001) { + if (input[8] > 1067.5000000000002) { + var34 = 0.041717597065890205; + } else { + var34 = -0.10357913492269129; + } + } else { + if (input[10] > 29.500000000000004) { + var34 = 0.1365512866715726; + } else { + var34 = 0.020600048310575665; + } + } + } + } + } else { + var34 = 0.09708785634773187; + } + } else { + var34 = -0.060427658852305666; + } + } else { + if (input[126] > 1e-35) { + if (input[10] > 32.50000000000001) { + if (input[6] > 24.500000000000004) { + if (input[8] > 1146.5000000000002) { + var34 = -0.03146213719547347; + } else { + var34 = 0.11784024316238083; + } + } else { + var34 = -0.050940520532045355; + } + } else { + var34 = -0.047988344143075616; + } + } else { + if (input[191] > 1e-35) { + var34 = 0.028764654731460032; + } else { + var34 = 0.0011911575567860023; + } + } + } + } + } + } + } + let var35: number; + if (input[294] > 1e-35) { + if (input[10] > 50.50000000000001) { + var35 = -0.11630092297244568; + } else { + if (input[0] > 2432.5000000000005) { + if (input[0] > 4199.500000000001) { + var35 = -0.05103908560370243; + } else { + var35 = 0.05002066201169583; + } + } else { + var35 = -0.09976646725732496; + } + } + } else { + if (input[32] > 1e-35) { + if (input[0] > 4242.500000000001) { + var35 = -0.0648838712201258; + } else { + if (input[5] > 3721.5000000000005) { + if (input[9] > 4.500000000000001) { + var35 = 0.127983140816313; + } else { + var35 = -0.05436534163636867; + } + } else { + var35 = -0.024514536544596455; + } + } + } else { + if (input[121] > 1e-35) { + if (input[0] > 4449.500000000001) { + if (input[4] > 9.500000000000002) { + var35 = -0.009504203657088933; + } else { + if (input[8] > 819.5000000000001) { + var35 = 0.18689664822602375; + } else { + var35 = 0.03635576744011826; + } + } + } else { + var35 = -0.029862411809998525; + } + } else { + if (input[223] > 1e-35) { + var35 = -0.06474496692999487; + } else { + if (input[86] > 1e-35) { + if (input[8] > 65.50000000000001) { + if (input[1] > 46.50000000000001) { + var35 = -0.09405026597863717; + } else { + if (input[0] > 4153.500000000001) { + var35 = 0.053577663326799765; + } else { + var35 = -0.05062127873995668; + } + } + } else { + var35 = 0.06512222894425874; + } + } else { + if (input[39] > 1e-35) { + var35 = -0.04985311717827547; + } else { + if (input[51] > 1e-35) { + var35 = -0.04541229517934797; + } else { + if (input[178] > 1e-35) { + if (input[2] > 25.500000000000004) { + if (input[2] > 30.500000000000004) { + if (input[0] > 2151.5000000000005) { + var35 = -0.02860634573675884; + } else { + var35 = 0.08863753005590103; + } + } else { + var35 = 0.11158892111063744; + } + } else { + if (input[0] > 655.5000000000001) { + var35 = -0.031005736641654926; + } else { + var35 = -0.1439827004505974; + } + } + } else { + if (input[222] > 1e-35) { + if (input[1] > 11.500000000000002) { + if (input[0] > 612.5000000000001) { + var35 = -0.00843386136334982; + } else { + var35 = -0.05273594615999777; + } + } else { + var35 = 0.1060183822015004; + } + } else { + if (input[126] > 1e-35) { + if (input[10] > 32.50000000000001) { + if (input[8] > 719.5000000000001) { + var35 = -0.015774115523598486; + } else { + var35 = 0.10147367091236065; + } + } else { + var35 = -0.048307000563071016; + } + } else { + var35 = 0.002118376117677254; + } + } + } + } + } + } + } + } + } + } + let var36: number; + if (input[8] > 1014.5000000000001) { + if (input[9] > 137.50000000000003) { + var36 = -0.10279096288817871; + } else { + if (input[0] > 93.50000000000001) { + if (input[8] > 1067.5000000000002) { + if (input[227] > 1e-35) { + var36 = -0.03544332389470493; + } else { + if (input[285] > 1e-35) { + if (input[9] > 64.50000000000001) { + var36 = 0.07211107542565391; + } else { + var36 = -0.041556776020476104; + } + } else { + if (input[145] > 1e-35) { + if (input[1] > 66.50000000000001) { + var36 = -0.0751486415451188; + } else { + if (input[1] > 59.50000000000001) { + var36 = 0.13459005084554104; + } else { + var36 = 0.024184371850147466; + } + } + } else { + if (input[0] > 3072.5000000000005) { + if (input[95] > 1e-35) { + var36 = 0.06715575425741895; + } else { + var36 = -0.005895690393702183; + } + } else { + if (input[8] > 2915.5000000000005) { + var36 = -0.010205039411753762; + } else { + if (input[9] > 33.50000000000001) { + if (input[9] > 47.50000000000001) { + var36 = -0.00029068886245881074; + } else { + var36 = 0.0613467393188786; + } + } else { + if (input[148] > 1e-35) { + var36 = -0.06074463294936236; + } else { + if (input[3] > 1.5000000000000002) { + if (input[5] > 1849.5000000000002) { + if (input[1] > 15.500000000000002) { + var36 = 0.003887223773199377; + } else { + var36 = -0.08553893131979015; + } + } else { + var36 = 0.025654192706396767; + } + } else { + var36 = -0.05651733979610658; + } + } + } + } + } + } + } + } + } else { + var36 = -0.02039913645229667; + } + } else { + if (input[2] > 7.500000000000001) { + var36 = -0.1058450646728524; + } else { + var36 = 0.02267192191610376; + } + } + } + } else { + if (input[1] > 120.50000000000001) { + if (input[2] > 60.50000000000001) { + var36 = -0.12304707569000428; + } else { + if (input[1] > 132.50000000000003) { + if (input[6] > 41.50000000000001) { + var36 = 0.1283258201586378; + } else { + var36 = -0.01718135372229775; + } + } else { + var36 = -0.07702452408491414; + } + } + } else { + if (input[125] > 1e-35) { + var36 = -0.0804612900572707; + } else { + if (input[178] > 1e-35) { + if (input[0] > 4533.500000000001) { + var36 = 0.04273051857848212; + } else { + var36 = -0.04533122948101463; + } + } else { + if (input[2] > 196.50000000000003) { + var36 = -0.10543331044088727; + } else { + if (input[94] > 1e-35) { + if (input[5] > 4532.500000000001) { + var36 = 0.0231032972703664; + } else { + var36 = -0.04807386814498683; + } + } else { + var36 = 0.002729435991332102; + } + } + } + } + } + } + let var37: number; + if (input[179] > 1e-35) { + var37 = -0.08065315471211375; + } else { + if (input[183] > 1e-35) { + if (input[17] > 1e-35) { + var37 = 0.026484626664041125; + } else { + if (input[10] > 1.5000000000000002) { + var37 = -0.10187000872941615; + } else { + var37 = 0.015274190652133752; + } + } + } else { + if (input[84] > 1e-35) { + if (input[9] > 6.500000000000001) { + if (input[2] > 43.50000000000001) { + var37 = 0.09574540795390041; + } else { + var37 = -0.06454986703691233; + } + } else { + var37 = -0.11411849349353141; + } + } else { + if (input[266] > 1e-35) { + var37 = -0.09281838517322076; + } else { + if (input[32] > 1e-35) { + if (input[8] > 2302.5000000000005) { + var37 = 0.06685250330182936; + } else { + if (input[4] > 67.50000000000001) { + if (input[2] > 97.50000000000001) { + var37 = -0.04403391373512386; + } else { + var37 = 0.1132928075412222; + } + } else { + if (input[2] > 47.50000000000001) { + var37 = -0.09700191391838056; + } else { + var37 = -0.02147184357182825; + } + } + } + } else { + if (input[10] > 4.500000000000001) { + if (input[21] > 1e-35) { + var37 = -0.0735617817957859; + } else { + if (input[17] > 1e-35) { + if (input[3] > 18.500000000000004) { + var37 = -0.001668912999010927; + } else { + var37 = -0.02363511102970245; + } + } else { + if (input[8] > 58.50000000000001) { + var37 = -0.00035213368294640616; + } else { + if (input[3] > 17.500000000000004) { + if (input[2] > 28.500000000000004) { + if (input[10] > 23.500000000000004) { + if (input[1] > 38.50000000000001) { + var37 = 0.0911011436534449; + } else { + if (input[1] > 28.500000000000004) { + var37 = -0.07192390493729035; + } else { + var37 = 0.06913818091291246; + } + } + } else { + var37 = -0.012312625373699222; + } + } else { + var37 = 0.06784496312307986; + } + } else { + var37 = -0.0000167756936027735; + } + } + } + } + } else { + if (input[18] > 1e-35) { + if (input[8] > 302.50000000000006) { + var37 = 0.0026564453057705273; + } else { + var37 = -0.025425772389361445; + } + } else { + if (input[122] > 1e-35) { + var37 = -0.12046786388602149; + } else { + if (input[0] > 3183.5000000000005) { + var37 = 0.01162092842804907; + } else { + if (input[91] > 1e-35) { + var37 = 0.07000265526928563; + } else { + if (input[1] > 22.500000000000004) { + if (input[0] > 576.5000000000001) { + var37 = -0.0001647792543020228; + } else { + var37 = -0.023664538532907665; + } + } else { + var37 = 0.01609078206180752; + } + } + } + } + } + } + } + } + } + } + } + let var38: number; + if (input[294] > 1e-35) { + if (input[1] > 26.500000000000004) { + if (input[0] > 4141.500000000001) { + var38 = -0.051473645433684705; + } else { + if (input[0] > 3030.5000000000005) { + if (input[1] > 51.50000000000001) { + var38 = -0.017696526862422682; + } else { + var38 = 0.1450050954613223; + } + } else { + var38 = -0.05406930069823832; + } + } + } else { + var38 = -0.08308700260259043; + } + } else { + if (input[120] > 1e-35) { + var38 = 0.058316269489189415; + } else { + if (input[297] > 1e-35) { + if (input[94] > 1e-35) { + var38 = -0.07425512495167255; + } else { + if (input[8] > 51.50000000000001) { + if (input[1] > 13.500000000000002) { + if (input[1] > 33.50000000000001) { + if (input[19] > 1e-35) { + if (input[0] > 4498.500000000001) { + var38 = 0.038431826961746934; + } else { + var38 = -0.05937462906539856; + } + } else { + if (input[9] > 65.50000000000001) { + var38 = 0.10814845712507865; + } else { + if (input[4] > 9.500000000000002) { + if (input[2] > 22.500000000000004) { + if (input[1] > 39.50000000000001) { + if (input[1] > 44.50000000000001) { + if (input[10] > 44.50000000000001) { + var38 = 0.12297945639231944; + } else { + if (input[0] > 3796.5000000000005) { + if (input[4] > 26.500000000000004) { + var38 = -0.09579030954062734; + } else { + var38 = 0.025064711572811746; + } + } else { + var38 = 0.02579440518821548; + } + } + } else { + var38 = 0.1044440128091862; + } + } else { + var38 = -0.058348633139536844; + } + } else { + var38 = 0.07766788227934436; + } + } else { + var38 = -0.01021229539092708; + } + } + } + } else { + if (input[2] > 2.5000000000000004) { + if (input[10] > 29.500000000000004) { + if (input[0] > 3770.5000000000005) { + if (input[0] > 4438.500000000001) { + var38 = 0.07463684068207214; + } else { + var38 = 0.18244269035484484; + } + } else { + if (input[6] > 39.50000000000001) { + var38 = -0.06050050067471004; + } else { + var38 = 0.05787759066913493; + } + } + } else { + var38 = 0.010783225857972171; + } + } else { + var38 = 0.1674891243602606; + } + } + } else { + if (input[4] > 9.500000000000002) { + var38 = -0.004814132027475892; + } else { + var38 = -0.14543299413454813; + } + } + } else { + var38 = -0.02935093398687923; + } + } + } else { + if (input[116] > 1e-35) { + if (input[9] > 2.5000000000000004) { + if (input[8] > 1218.5000000000002) { + var38 = -0.07634466313617769; + } else { + var38 = 0.0287825335169114; + } + } else { + var38 = -0.06894721943300268; + } + } else { + var38 = -0.00023988459059521937; + } + } + } + } + let var39: number; + if (input[131] > 1e-35) { + if (input[1] > 93.50000000000001) { + var39 = -0.05706887458825395; + } else { + if (input[2] > 1.5000000000000002) { + var39 = 0.011446637886629108; + } else { + var39 = -0.10616119878749211; + } + } + } else { + if (input[230] > 1e-35) { + if (input[4] > 6.500000000000001) { + if (input[0] > 4977.500000000001) { + var39 = 0.08424281276381033; + } else { + if (input[3] > 17.500000000000004) { + if (input[20] > 1e-35) { + var39 = 0.11146885439601915; + } else { + if (input[8] > 61.50000000000001) { + if (input[0] > 3530.5000000000005) { + if (input[9] > 48.50000000000001) { + if (input[9] > 61.50000000000001) { + var39 = 0.026278724448495064; + } else { + var39 = 0.17053138400480508; + } + } else { + if (input[0] > 4463.500000000001) { + var39 = -0.06482289890096041; + } else { + var39 = 0.03026516489536295; + } + } + } else { + var39 = -0.031785170717683144; + } + } else { + var39 = 0.1312690622980455; + } + } + } else { + if (input[13] > 1e-35) { + var39 = 0.14336922540461444; + } else { + var39 = 0.03523850945454039; + } + } + } + } else { + var39 = -0.015407465968975714; + } + } else { + if (input[39] > 1e-35) { + var39 = -0.054809635385158186; + } else { + if (input[32] > 1e-35) { + if (input[0] > 4242.500000000001) { + var39 = -0.0659975068798723; + } else { + var39 = -0.008386582621403979; + } + } else { + if (input[4] > 60.50000000000001) { + if (input[10] > 75.50000000000001) { + if (input[3] > 107.50000000000001) { + var39 = -0.04225314193574262; + } else { + if (input[3] > 70.50000000000001) { + if (input[1] > 29.500000000000004) { + var39 = 0.057409156184759516; + } else { + var39 = 0.2024322059866388; + } + } else { + var39 = -0.030670938454461245; + } + } + } else { + if (input[10] > 1e-35) { + if (input[0] > 4733.500000000001) { + var39 = 0.010648654146284154; + } else { + if (input[308] > 1e-35) { + var39 = 0.008728141696325391; + } else { + if (input[4] > 64.50000000000001) { + if (input[298] > 1e-35) { + var39 = 0.12364025998551711; + } else { + var39 = -0.02247495081065243; + } + } else { + if (input[1] > 22.500000000000004) { + var39 = -0.0726295464624251; + } else { + var39 = 0.03481895086048152; + } + } + } + } + } else { + if (input[0] > 4331.500000000001) { + var39 = -0.04775443357020673; + } else { + var39 = 0.07172377425057568; + } + } + } + } else { + if (input[2] > 89.50000000000001) { + var39 = -0.11782645274716962; + } else { + var39 = 0.00010092665257989378; + } + } + } + } + } + } + let var40: number; + if (input[147] > 1e-35) { + var40 = -0.041560228567115574; + } else { + if (input[302] > 1e-35) { + if (input[10] > 47.50000000000001) { + var40 = 0.062292114082780084; + } else { + if (input[10] > 5.500000000000001) { + if (input[7] > 22.500000000000004) { + var40 = -0.016101990375700172; + } else { + if (input[0] > 2579.5000000000005) { + var40 = -0.13045089661551845; + } else { + var40 = -0.02874367814784938; + } + } + } else { + var40 = 0.025835149631944995; + } + } + } else { + if (input[167] > 1e-35) { + if (input[0] > 3928.5000000000005) { + var40 = 0.17084176915326055; + } else { + var40 = -0.019195947948312853; + } + } else { + if (input[222] > 1e-35) { + if (input[30] > 1e-35) { + if (input[1] > 36.50000000000001) { + if (input[8] > 45.50000000000001) { + if (input[8] > 578.5000000000001) { + if (input[1] > 67.50000000000001) { + var40 = 0.10591712319944074; + } else { + var40 = -0.024082167264285; + } + } else { + var40 = 0.16497698867036126; + } + } else { + var40 = -0.04985066326861431; + } + } else { + if (input[0] > 1937.5000000000002) { + if (input[2] > 16.500000000000004) { + var40 = -0.021012910475524206; + } else { + var40 = -0.13058422554298485; + } + } else { + if (input[0] > 1102.5000000000002) { + var40 = 0.10955864175201457; + } else { + var40 = -0.03566689354348996; + } + } + } + } else { + if (input[1] > 11.500000000000002) { + var40 = -0.02093884208606101; + } else { + var40 = 0.09107244766183857; + } + } + } else { + if (input[126] > 1e-35) { + if (input[10] > 32.50000000000001) { + if (input[8] > 719.5000000000001) { + var40 = -0.013861861436128482; + } else { + var40 = 0.09756849802202777; + } + } else { + if (input[224] > 1e-35) { + if (input[1] > 51.50000000000001) { + var40 = 0.10163873449625677; + } else { + var40 = -0.02779270277623805; + } + } else { + if (input[1] > 26.500000000000004) { + var40 = -0.08035058228527389; + } else { + var40 = 0.0005719695099064484; + } + } + } + } else { + if (input[191] > 1e-35) { + if (input[9] > 9.500000000000002) { + var40 = -0.007028075523033826; + } else { + var40 = 0.0489470913925288; + } + } else { + if (input[1] > 61.50000000000001) { + if (input[132] > 1e-35) { + var40 = 0.11230846723576784; + } else { + if (input[0] > 350.50000000000006) { + if (input[2] > 1.5000000000000002) { + var40 = -0.0032075580718124892; + } else { + var40 = -0.04442829143298883; + } + } else { + var40 = -0.06597073245775804; + } + } + } else { + var40 = 0.0015594090939337751; + } + } + } + } + } + } + } + let var41: number; + if (input[223] > 1e-35) { + if (input[8] > 668.5000000000001) { + var41 = -0.12803889879260094; + } else { + var41 = 0.002171373740016862; + } + } else { + if (input[121] > 1e-35) { + if (input[0] > 4720.500000000001) { + if (input[217] > 1e-35) { + var41 = 0.08967966612917375; + } else { + if (input[1] > 39.50000000000001) { + var41 = -0.059791671514498074; + } else { + var41 = 0.05648934961902822; + } + } + } else { + if (input[2] > 59.50000000000001) { + var41 = -0.08633234097449628; + } else { + if (input[6] > 53.50000000000001) { + var41 = 0.11140345067444689; + } else { + if (input[1] > 56.50000000000001) { + if (input[4] > 7.500000000000001) { + if (input[0] > 3560.5000000000005) { + var41 = 0.025606129643140924; + } else { + var41 = 0.13835395886271978; + } + } else { + var41 = -0.09361630641448024; + } + } else { + if (input[4] > 7.500000000000001) { + if (input[1] > 26.500000000000004) { + if (input[1] > 49.50000000000001) { + var41 = -0.09975506556937946; + } else { + if (input[10] > 36.50000000000001) { + var41 = -0.09427724661655643; + } else { + if (input[10] > 24.500000000000004) { + var41 = 0.07329330653410447; + } else { + var41 = -0.02271182965807972; + } + } + } + } else { + var41 = -0.09767874967639482; + } + } else { + if (input[6] > 13.500000000000002) { + if (input[10] > 23.500000000000004) { + var41 = -0.05082091374050816; + } else { + var41 = 0.1687114435254966; + } + } else { + if (input[0] > 2314.5000000000005) { + var41 = -0.06422664016383926; + } else { + var41 = 0.0636688376664789; + } + } + } + } + } + } + } + } else { + if (input[298] > 1e-35) { + if (input[9] > 12.500000000000002) { + if (input[133] > 1e-35) { + var41 = -0.06857762517406195; + } else { + if (input[9] > 71.50000000000001) { + if (input[0] > 4188.500000000001) { + var41 = -0.1274167728754332; + } else { + var41 = 0.01308079126447365; + } + } else { + if (input[4] > 73.50000000000001) { + var41 = 0.13854015371106546; + } else { + if (input[4] > 48.50000000000001) { + var41 = -0.03684255740123261; + } else { + if (input[6] > 45.50000000000001) { + var41 = 0.10329912215813097; + } else { + if (input[10] > 77.50000000000001) { + var41 = -0.08630788656925215; + } else { + var41 = 0.031022006843800853; + } + } + } + } + } + } + } else { + if (input[1] > 25.500000000000004) { + var41 = -0.08278381528048026; + } else { + var41 = 0.06664374548141594; + } + } + } else { + if (input[84] > 1e-35) { + var41 = -0.05624227409079396; + } else { + var41 = 0.00012184182357340415; + } + } + } + } + let var42: number; + if (input[179] > 1e-35) { + var42 = -0.07443348719246982; + } else { + if (input[40] > 1e-35) { + if (input[0] > 1937.5000000000002) { + var42 = -0.07595415373151816; + } else { + var42 = 0.054065040429292326; + } + } else { + if (input[134] > 1e-35) { + if (input[11] > 1e-35) { + if (input[2] > 13.500000000000002) { + if (input[0] > 1187.5000000000002) { + var42 = 0.022822510448266862; + } else { + var42 = 0.17491569312933697; + } + } else { + var42 = -0.058362287133533565; + } + } else { + if (input[2] > 2.5000000000000004) { + var42 = -0.03633895806364428; + } else { + var42 = 0.06397808186120692; + } + } + } else { + if (input[8] > 4968.500000000001) { + if (input[1] > 31.500000000000004) { + var42 = -0.07294848747514579; + } else { + var42 = 0.025053613105805606; + } + } else { + if (input[230] > 1e-35) { + if (input[4] > 6.500000000000001) { + if (input[107] > 1e-35) { + var42 = -0.07009535282685533; + } else { + if (input[8] > 2640.0000000000005) { + var42 = -0.051761240111316276; + } else { + if (input[131] > 1e-35) { + var42 = -0.06245774419231631; + } else { + var42 = 0.03495606662854905; + } + } + } + } else { + var42 = -0.013863522184803188; + } + } else { + if (input[131] > 1e-35) { + if (input[1] > 93.50000000000001) { + if (input[1] > 105.50000000000001) { + var42 = 0.0015036626973581122; + } else { + var42 = -0.12505706794835883; + } + } else { + if (input[1] > 48.50000000000001) { + if (input[276] > 1e-35) { + var42 = 0.10435171369790015; + } else { + if (input[0] > 5026.500000000001) { + if (input[0] > 5308.500000000001) { + var42 = 0.022343994371919224; + } else { + var42 = -0.14087991797693533; + } + } else { + if (input[8] > 1323.5000000000002) { + if (input[10] > 49.50000000000001) { + var42 = 0.07724450228328664; + } else { + if (input[0] > 3853.5000000000005) { + var42 = -0.15671707454435677; + } else { + if (input[10] > 28.500000000000004) { + var42 = -0.10179090671841723; + } else { + var42 = 0.014878216919760927; + } + } + } + } else { + var42 = 0.03967665658164865; + } + } + } + } else { + if (input[8] > 2696.5000000000005) { + if (input[15] > 1e-35) { + var42 = 0.14054154485273487; + } else { + var42 = 0.01821247272493051; + } + } else { + if (input[2] > 5.500000000000001) { + if (input[2] > 100.50000000000001) { + var42 = -0.08632985141410315; + } else { + var42 = 0.005524157938954954; + } + } else { + var42 = -0.08802502622523681; + } + } + } + } + } else { + var42 = -0.0004649168897260341; + } + } + } + } + } + } + let var43: number; + if (input[86] > 1e-35) { + if (input[8] > 65.50000000000001) { + if (input[1] > 32.50000000000001) { + if (input[4] > 16.500000000000004) { + var43 = -0.007458687464321174; + } else { + var43 = -0.09444966249102484; + } + } else { + if (input[1] > 23.500000000000004) { + var43 = 0.08564129697360716; + } else { + var43 = -0.07105002902845851; + } + } + } else { + var43 = 0.05688756955238231; + } + } else { + if (input[294] > 1e-35) { + if (input[10] > 50.50000000000001) { + var43 = -0.10326216566705966; + } else { + if (input[1] > 26.500000000000004) { + var43 = 0.0050539832484585365; + } else { + var43 = -0.07080395606126953; + } + } + } else { + if (input[306] > 1e-35) { + if (input[149] > 1e-35) { + var43 = -0.10399433201474328; + } else { + if (input[2] > 14.500000000000002) { + if (input[9] > 6.500000000000001) { + var43 = 0.05783632021087773; + } else { + if (input[10] > 17.500000000000004) { + var43 = -0.06720598671764105; + } else { + if (input[1] > 47.50000000000001) { + var43 = 0.097495825172558; + } else { + var43 = -0.013372242800584872; + } + } + } + } else { + var43 = -0.06463226787713715; + } + } + } else { + if (input[42] > 1e-35) { + var43 = -0.0885725817597767; + } else { + if (input[204] > 1e-35) { + if (input[1] > 62.50000000000001) { + var43 = -0.07496598696848249; + } else { + if (input[1] > 29.500000000000004) { + if (input[8] > 446.50000000000006) { + var43 = 0.11051270080118503; + } else { + var43 = 0.027719462817590454; + } + } else { + if (input[8] > 597.5000000000001) { + var43 = -0.08441503592016869; + } else { + var43 = 0.05534229430302502; + } + } + } + } else { + if (input[223] > 1e-35) { + if (input[8] > 668.5000000000001) { + var43 = -0.12190088985091102; + } else { + var43 = -0.0067442838156576345; + } + } else { + if (input[148] > 1e-35) { + if (input[9] > 79.50000000000001) { + var43 = 0.09225972475904022; + } else { + if (input[2] > 10.500000000000002) { + if (input[1] > 102.50000000000001) { + var43 = 0.11805676536334647; + } else { + if (input[8] > 1726.5000000000002) { + if (input[9] > 10.500000000000002) { + var43 = 0.016585157185448045; + } else { + var43 = -0.11032043771149425; + } + } else { + var43 = 0.01586986028570486; + } + } + } else { + if (input[8] > 388.50000000000006) { + var43 = -0.10592413013261853; + } else { + var43 = 0.04930703248769364; + } + } + } + } else { + if (input[13] > 1e-35) { + var43 = 0.003621937787920821; + } else { + var43 = -0.0013786331198611841; + } + } + } + } + } + } + } + } + let var44: number; + if (input[145] > 1e-35) { + if (input[1] > 32.50000000000001) { + if (input[1] > 38.50000000000001) { + if (input[10] > 55.50000000000001) { + if (input[1] > 54.50000000000001) { + var44 = 0.009769895322846493; + } else { + var44 = -0.10620052926943656; + } + } else { + if (input[9] > 19.500000000000004) { + var44 = 0.03781202525403449; + } else { + if (input[9] > 14.500000000000002) { + var44 = -0.11485785321365344; + } else { + if (input[9] > 6.500000000000001) { + var44 = 0.07677177833073881; + } else { + if (input[0] > 4342.500000000001) { + var44 = -0.07079285609687631; + } else { + if (input[49] > 1e-35) { + var44 = 0.06156814809246001; + } else { + var44 = -0.014788509042554625; + } + } + } + } + } + } + } else { + var44 = -0.032659201618470655; + } + } else { + if (input[5] > 5207.500000000001) { + var44 = -0.09013500825185713; + } else { + if (input[3] > 10.500000000000002) { + if (input[8] > 1787.5000000000002) { + var44 = -0.03094160322187924; + } else { + if (input[1] > 29.500000000000004) { + var44 = 0.09474646043921069; + } else { + var44 = 0.023445783928231618; + } + } + } else { + var44 = 0.09342846694174194; + } + } + } + } else { + if (input[0] > 533.5000000000001) { + if (input[204] > 1e-35) { + if (input[1] > 62.50000000000001) { + var44 = -0.07164443768784848; + } else { + if (input[1] > 29.500000000000004) { + var44 = 0.089473622509272; + } else { + if (input[8] > 597.5000000000001) { + var44 = -0.08155349903101317; + } else { + var44 = 0.07098423265024251; + } + } + } + } else { + if (input[8] > 691.5000000000001) { + if (input[5] > 2252.5000000000005) { + var44 = -0.004003900679358653; + } else { + if (input[190] > 1e-35) { + var44 = -0.09236113461485262; + } else { + if (input[8] > 3198.5000000000005) { + var44 = -0.0124130160451179; + } else { + var44 = 0.018453070064009328; + } + } + } + } else { + if (input[15] > 1e-35) { + var44 = 0.012013209112857824; + } else { + if (input[7] > 4.500000000000001) { + if (input[7] > 5.500000000000001) { + var44 = -0.0009580759587680961; + } else { + var44 = -0.03227283036698222; + } + } else { + var44 = 0.01369287669536875; + } + } + } + } + } else { + if (input[1] > 50.50000000000001) { + var44 = -0.04213060332500437; + } else { + if (input[35] > 1e-35) { + var44 = -0.11508095777767471; + } else { + if (input[190] > 1e-35) { + var44 = -0.08611884672400155; + } else { + if (input[297] > 1e-35) { + var44 = 0.05723551879433584; + } else { + var44 = -0.004829340082311461; + } + } + } + } + } + } + let var45: number; + if (input[183] > 1e-35) { + var45 = -0.037994150023203555; + } else { + if (input[227] > 1e-35) { + if (input[17] > 1e-35) { + if (input[3] > 20.500000000000004) { + if (input[10] > 36.50000000000001) { + var45 = -0.11753465135886734; + } else { + var45 = -0.007515490299047085; + } + } else { + var45 = -0.08576941990777916; + } + } else { + if (input[8] > 1641.5000000000002) { + if (input[10] > 37.50000000000001) { + var45 = -0.12371142493530439; + } else { + if (input[1] > 36.50000000000001) { + var45 = 0.032189417575190435; + } else { + var45 = -0.10339125953022954; + } + } + } else { + if (input[3] > 32.50000000000001) { + if (input[4] > 27.500000000000004) { + if (input[1] > 59.50000000000001) { + var45 = -0.0784518658439288; + } else { + if (input[2] > 54.50000000000001) { + var45 = 0.12477882322370665; + } else { + var45 = 0.000313468482399738; + } + } + } else { + var45 = 0.12261955132611434; + } + } else { + if (input[8] > 81.50000000000001) { + if (input[23] > 1e-35) { + var45 = 0.04969252946760318; + } else { + if (input[8] > 511.50000000000006) { + if (input[8] > 1146.5000000000002) { + var45 = 0.0353146070135579; + } else { + var45 = -0.06327619611098285; + } + } else { + var45 = 0.02813577701641991; + } + } + } else { + var45 = -0.12354390728506215; + } + } + } + } + } else { + if (input[34] > 1e-35) { + var45 = -0.07664408516055397; + } else { + if (input[3] > 99.50000000000001) { + if (input[1] > 16.500000000000004) { + if (input[1] > 26.500000000000004) { + var45 = -0.01245803535276381; + } else { + var45 = -0.07169472553475001; + } + } else { + if (input[1] > 11.500000000000002) { + var45 = 0.12989984824561698; + } else { + var45 = -0.01201544398886606; + } + } + } else { + if (input[6] > 91.50000000000001) { + if (input[1] > 22.500000000000004) { + var45 = 0.010390226893521422; + } else { + if (input[10] > 14.500000000000002) { + var45 = 0.16790888126487719; + } else { + var45 = 0.010614982228955577; + } + } + } else { + if (input[4] > 79.50000000000001) { + if (input[9] > 44.50000000000001) { + if (input[0] > 3853.5000000000005) { + var45 = -0.043398307129729134; + } else { + var45 = 0.09963544907820426; + } + } else { + if (input[9] > 30.500000000000004) { + var45 = -0.13540713124984502; + } else { + if (input[9] > 17.500000000000004) { + var45 = 0.0509435850590757; + } else { + var45 = -0.04761897852404613; + } + } + } + } else { + if (input[4] > 78.50000000000001) { + var45 = 0.09197086656470652; + } else { + var45 = 0.0006771050176682337; + } + } + } + } + } + } + } + let var46: number; + if (input[122] > 1e-35) { + if (input[6] > 36.50000000000001) { + var46 = 0.05686884451670743; + } else { + var46 = -0.05334759543084309; + } + } else { + if (input[266] > 1e-35) { + var46 = -0.08603579519816038; + } else { + if (input[157] > 1e-35) { + var46 = -0.06736746113382097; + } else { + if (input[302] > 1e-35) { + if (input[0] > 2579.5000000000005) { + var46 = -0.0499592651503952; + } else { + if (input[0] > 725.5000000000001) { + var46 = 0.11780353905132664; + } else { + var46 = -0.05232097173108943; + } + } + } else { + if (input[147] > 1e-35) { + if (input[1] > 53.50000000000001) { + var46 = -0.11398297342629615; + } else { + if (input[0] > 2604.5000000000005) { + if (input[0] > 3629.5000000000005) { + var46 = -0.03190157229022304; + } else { + var46 = 0.07985197845805492; + } + } else { + var46 = -0.0763078988943886; + } + } + } else { + if (input[4] > 41.50000000000001) { + if (input[280] > 1e-35) { + var46 = 0.05162933940904835; + } else { + if (input[11] > 1e-35) { + if (input[0] > 460.50000000000006) { + var46 = -0.027174047777029083; + } else { + var46 = 0.057117284879796476; + } + } else { + if (input[3] > 43.50000000000001) { + var46 = -0.0016147040913107311; + } else { + var46 = -0.05856597304613519; + } + } + } + } else { + if (input[2] > 45.50000000000001) { + if (input[0] > 4663.500000000001) { + if (input[18] > 1e-35) { + var46 = -0.04779247091640426; + } else { + if (input[10] > 25.500000000000004) { + if (input[9] > 22.500000000000004) { + if (input[22] > 1e-35) { + var46 = -0.01466076988151239; + } else { + var46 = 0.13375695925484857; + } + } else { + var46 = -0.04885873081899647; + } + } else { + if (input[0] > 5566.500000000001) { + var46 = 0.11086813028591343; + } else { + if (input[8] > 992.5000000000001) { + var46 = -0.07622304217072383; + } else { + var46 = 0.04316019272026325; + } + } + } + } + } else { + if (input[10] > 12.500000000000002) { + if (input[9] > 36.50000000000001) { + if (input[9] > 45.50000000000001) { + var46 = 0.03285858361708423; + } else { + var46 = -0.12354858211764992; + } + } else { + var46 = 0.0672788301823281; + } + } else { + if (input[15] > 1e-35) { + var46 = 0.08658836986585006; + } else { + var46 = -0.02741484278509758; + } + } + } + } else { + if (input[290] > 1e-35) { + var46 = -0.08161310335133287; + } else { + if (input[135] > 1e-35) { + var46 = -0.04824156054814152; + } else { + var46 = 0.0009156904299554183; + } + } + } + } + } + } + } + } + } + let var47: number; + if (input[3] > 7.500000000000001) { + var47 = 0.0006791852818377787; + } else { + if (input[129] > 1e-35) { + if (input[0] > 2904.5000000000005) { + if (input[0] > 4004.5000000000005) { + var47 = 0.03642374718166293; + } else { + var47 = 0.16379973756366603; + } + } else { + var47 = -0.03946685266127979; + } + } else { + if (input[186] > 1e-35) { + var47 = 0.07618896623420895; + } else { + if (input[96] > 1e-35) { + var47 = 0.0680272261319657; + } else { + if (input[107] > 1e-35) { + if (input[1] > 48.50000000000001) { + var47 = -0.022822371600847505; + } else { + var47 = 0.0501405836324949; + } + } else { + if (input[203] > 1e-35) { + if (input[1] > 77.50000000000001) { + var47 = 0.044416424920571296; + } else { + var47 = -0.0648450593196238; + } + } else { + if (input[5] > 3921.5000000000005) { + if (input[1] > 110.50000000000001) { + var47 = -0.11110466767595227; + } else { + if (input[9] > 5.500000000000001) { + if (input[9] > 52.50000000000001) { + if (input[1] > 50.50000000000001) { + var47 = 0.1061937286809567; + } else { + if (input[7] > 54.50000000000001) { + var47 = 0.11487507743121311; + } else { + if (input[8] > 819.5000000000001) { + var47 = -0.07181278009001418; + } else { + if (input[10] > 25.500000000000004) { + var47 = 0.13499019430369633; + } else { + if (input[1] > 31.500000000000004) { + var47 = 0.09032979489780704; + } else { + var47 = -0.12754166393372374; + } + } + } + } + } + } else { + if (input[9] > 37.50000000000001) { + var47 = -0.05093963635361407; + } else { + var47 = -0.005026651151683848; + } + } + } else { + if (input[9] > 2.5000000000000004) { + var47 = 0.07619735785573735; + } else { + var47 = 0.012363301341532136; + } + } + } + } else { + if (input[26] > 1e-35) { + var47 = -0.10685800454968203; + } else { + if (input[8] > 125.50000000000001) { + if (input[8] > 446.50000000000006) { + if (input[0] > 3842.5000000000005) { + var47 = -0.08783796894105043; + } else { + if (input[282] > 1e-35) { + if (input[1] > 47.50000000000001) { + if (input[9] > 40.50000000000001) { + var47 = -0.10764172927882483; + } else { + var47 = 0.01890760098464703; + } + } else { + var47 = 0.06573095405846417; + } + } else { + if (input[8] > 634.5000000000001) { + var47 = -0.00783575973273707; + } else { + var47 = -0.050612689680229306; + } + } + } + } else { + if (input[1] > 22.500000000000004) { + var47 = -0.0016842490401359626; + } else { + var47 = 0.0738227088444087; + } + } + } else { + var47 = -0.02663970950432175; + } + } + } + } + } + } + } + } + } + let var48: number; + if (input[31] > 1e-35) { + if (input[8] > 17.500000000000004) { + var48 = 0.013678038624884814; + } else { + if (input[1] > 35.50000000000001) { + if (input[1] > 51.50000000000001) { + var48 = 0.007191286124908192; + } else { + var48 = -0.09347881647636902; + } + } else { + if (input[10] > 1.5000000000000002) { + var48 = 0.07938758708008091; + } else { + var48 = -0.008702935600305113; + } + } + } + } else { + if (input[224] > 1e-35) { + if (input[149] > 1e-35) { + if (input[13] > 1e-35) { + var48 = 0.12321804057595996; + } else { + var48 = -0.018281109320672437; + } + } else { + if (input[23] > 1e-35) { + if (input[4] > 62.50000000000001) { + var48 = -0.04644244754790671; + } else { + var48 = 0.024546310702263208; + } + } else { + if (input[8] > 862.5000000000001) { + if (input[0] > 3429.5000000000005) { + if (input[4] > 9.500000000000002) { + if (input[52] > 1e-35) { + var48 = 0.0706108609273337; + } else { + if (input[2] > 40.50000000000001) { + var48 = -0.028046629962303716; + } else { + var48 = -0.06497613993109329; + } + } + } else { + var48 = 0.01076489668586676; + } + } else { + if (input[1] > 33.50000000000001) { + if (input[0] > 966.5000000000001) { + if (input[2] > 14.500000000000002) { + if (input[1] > 38.50000000000001) { + var48 = -0.03056331974267756; + } else { + var48 = -0.11886389712497057; + } + } else { + var48 = 0.053364962175658184; + } + } else { + if (input[8] > 2233.5000000000005) { + var48 = -0.0448152521157682; + } else { + var48 = 0.1508651602190868; + } + } + } else { + if (input[2] > 33.50000000000001) { + if (input[0] > 2882.5000000000005) { + if (input[0] > 3183.5000000000005) { + var48 = 0.03818796510453344; + } else { + var48 = 0.23673992112982362; + } + } else { + var48 = 0.02858814226507374; + } + } else { + if (input[10] > 44.50000000000001) { + var48 = -0.1125863771551199; + } else { + var48 = 0.009129996952394916; + } + } + } + } + } else { + if (input[1] > 7.500000000000001) { + var48 = -0.004374525302461639; + } else { + var48 = -0.07858519434925451; + } + } + } + } + } else { + if (input[149] > 1e-35) { + if (input[6] > 23.500000000000004) { + var48 = 0.0005231594491642136; + } else { + if (input[0] > 4053.5000000000005) { + if (input[8] > 660.5000000000001) { + var48 = -0.13677189943034931; + } else { + if (input[10] > 2.5000000000000004) { + var48 = 0.039591891437078086; + } else { + var48 = -0.09312596849507347; + } + } + } else { + var48 = -0.02423172142089822; + } + } + } else { + var48 = 0.0009836986075266283; + } + } + } + let var49: number; + if (input[189] > 1e-35) { + if (input[0] > 5269.500000000001) { + var49 = -0.103183298350443; + } else { + if (input[2] > 51.50000000000001) { + var49 = 0.09784373530929913; + } else { + if (input[10] > 26.500000000000004) { + if (input[8] > 764.5000000000001) { + var49 = -0.05186168947388339; + } else { + var49 = 0.0496996365539082; + } + } else { + if (input[10] > 23.500000000000004) { + var49 = 0.1404445738719; + } else { + if (input[93] > 1e-35) { + var49 = 0.0027146310074558505; + } else { + if (input[5] > 3821.5000000000005) { + var49 = 0.002153033152069652; + } else { + if (input[4] > 2.5000000000000004) { + var49 = 0.007663539551317215; + } else { + var49 = 0.13902616832015402; + } + } + } + } + } + } + } + } else { + if (input[298] > 1e-35) { + if (input[8] > 81.50000000000001) { + if (input[4] > 64.50000000000001) { + var49 = 0.11498405722487515; + } else { + if (input[2] > 23.500000000000004) { + if (input[0] > 2815.5000000000005) { + if (input[2] > 44.50000000000001) { + if (input[4] > 42.50000000000001) { + var49 = -0.021479467709980358; + } else { + var49 = 0.09336868994327292; + } + } else { + if (input[1] > 22.500000000000004) { + if (input[15] > 1e-35) { + var49 = 0.021660293256233334; + } else { + var49 = -0.0927396152303864; + } + } else { + var49 = 0.0665074081601698; + } + } + } else { + if (input[0] > 1550.5000000000002) { + var49 = 0.08972407105958534; + } else { + var49 = -0.0380796411182682; + } + } + } else { + if (input[6] > 13.500000000000002) { + if (input[10] > 2.5000000000000004) { + var49 = 0.06761927942466854; + } else { + var49 = -0.015762168112653286; + } + } else { + if (input[17] > 1e-35) { + var49 = 0.10311304131145381; + } else { + var49 = -0.017672785252336027; + } + } + } + } + } else { + var49 = -0.08629805732772755; + } + } else { + if (input[1] > 24.500000000000004) { + if (input[138] > 1e-35) { + var49 = -0.10638321435298535; + } else { + var49 = 0.0007073011744385905; + } + } else { + if (input[18] > 1e-35) { + var49 = -0.027056185501334325; + } else { + if (input[145] > 1e-35) { + var49 = 0.023191199677450886; + } else { + if (input[9] > 33.50000000000001) { + if (input[201] > 1e-35) { + var49 = 0.09762140519655171; + } else { + if (input[9] > 110.50000000000001) { + var49 = -0.06581942957595835; + } else { + if (input[6] > 54.50000000000001) { + var49 = 0.04959634035251596; + } else { + var49 = 0.0022616298654554207; + } + } + } + } else { + var49 = -0.007437620924990854; + } + } + } + } + } + } + let var50: number; + if (input[179] > 1e-35) { + var50 = -0.06961998209988884; + } else { + if (input[167] > 1e-35) { + if (input[0] > 3928.5000000000005) { + var50 = 0.1470294450403005; + } else { + var50 = -0.01671476793947083; + } + } else { + if (input[187] > 1e-35) { + if (input[6] > 13.500000000000002) { + if (input[4] > 30.500000000000004) { + if (input[13] > 1e-35) { + var50 = 0.07448480853603114; + } else { + if (input[0] > 1012.5000000000001) { + if (input[5] > 2883.5000000000005) { + if (input[0] > 3682.5000000000005) { + if (input[5] > 4031.5000000000005) { + if (input[23] > 1e-35) { + var50 = 0.07965955447707423; + } else { + if (input[10] > 10.500000000000002) { + var50 = -0.09236156404262426; + } else { + var50 = 0.03396273196231458; + } + } + } else { + var50 = -0.13246465021467432; + } + } else { + var50 = 0.07092822261735353; + } + } else { + var50 = -0.08753829085942; + } + } else { + var50 = 0.09409024840640956; + } + } + } else { + if (input[1] > 40.50000000000001) { + if (input[8] > 984.5000000000001) { + if (input[8] > 1514.5000000000002) { + if (input[8] > 2134.5000000000005) { + var50 = 0.004705878789890202; + } else { + var50 = 0.13775378964952867; + } + } else { + var50 = -0.04770928980587811; + } + } else { + if (input[10] > 29.500000000000004) { + var50 = 0.011221519891071544; + } else { + if (input[0] > 3853.5000000000005) { + var50 = 0.06365381191628273; + } else { + var50 = 0.15506252245336827; + } + } + } + } else { + if (input[1] > 37.50000000000001) { + var50 = -0.07254777021042061; + } else { + var50 = 0.026514587757252385; + } + } + } + } else { + if (input[308] > 1e-35) { + var50 = 0.04115804816617256; + } else { + if (input[10] > 26.500000000000004) { + var50 = 0.02077721353011946; + } else { + if (input[5] > 3548.5000000000005) { + var50 = -0.1280907116663952; + } else { + var50 = -0.021974774274438; + } + } + } + } + } else { + if (input[306] > 1e-35) { + var50 = -0.02700446558079895; + } else { + if (input[297] > 1e-35) { + if (input[212] > 1e-35) { + var50 = 0.07794139136748461; + } else { + if (input[7] > 5.500000000000001) { + if (input[19] > 1e-35) { + var50 = -0.005710865560475598; + } else { + if (input[94] > 1e-35) { + var50 = -0.06751507982853555; + } else { + var50 = 0.027250040757588703; + } + } + } else { + if (input[9] > 52.50000000000001) { + var50 = 0.07060357924595577; + } else { + var50 = -0.030297760713011795; + } + } + } + } else { + var50 = -0.0006005400085266517; + } + } + } + } + } + let var51: number; + if (input[113] > 1e-35) { + var51 = -0.07311041707507712; + } else { + if (input[40] > 1e-35) { + if (input[0] > 1937.5000000000002) { + var51 = -0.06996356565314456; + } else { + var51 = 0.04780211300352931; + } + } else { + if (input[10] > 52.50000000000001) { + if (input[49] > 1e-35) { + var51 = -0.08317707559926495; + } else { + if (input[21] > 1e-35) { + var51 = -0.0817284654645976; + } else { + if (input[15] > 1e-35) { + if (input[2] > 3.5000000000000004) { + var51 = -0.010538203005984922; + } else { + var51 = 0.08454819465349446; + } + } else { + if (input[9] > 124.50000000000001) { + var51 = 0.09015659250299132; + } else { + if (input[7] > 15.500000000000002) { + if (input[5] > 5732.500000000001) { + var51 = -0.08542251249346582; + } else { + if (input[9] > 50.50000000000001) { + var51 = -0.023428882537657472; + } else { + var51 = 0.010042500833979073; + } + } + } else { + var51 = 0.020697210754240154; + } + } + } + } + } + } else { + if (input[10] > 28.500000000000004) { + if (input[5] > 423.00000000000006) { + if (input[148] > 1e-35) { + var51 = 0.03006025206979096; + } else { + if (input[9] > 108.50000000000001) { + var51 = -0.09153851322499747; + } else { + if (input[145] > 1e-35) { + if (input[5] > 4814.500000000001) { + if (input[2] > 38.50000000000001) { + var51 = 0.04222035773042132; + } else { + var51 = -0.09078149053947535; + } + } else { + if (input[8] > 568.5000000000001) { + if (input[1] > 64.50000000000001) { + var51 = -0.07209095448054853; + } else { + var51 = 0.028065954981903313; + } + } else { + var51 = 0.08714651929917122; + } + } + } else { + var51 = -0.006678820669279169; + } + } + } + } else { + if (input[10] > 40.50000000000001) { + var51 = 0.006982396294941626; + } else { + var51 = -0.07889649792011418; + } + } + } else { + if (input[94] > 1e-35) { + if (input[4] > 30.500000000000004) { + var51 = -0.09351114982645548; + } else { + if (input[4] > 3.5000000000000004) { + var51 = -0.004837550129223451; + } else { + var51 = -0.08324141237464677; + } + } + } else { + if (input[303] > 1e-35) { + var51 = 0.10703037493990825; + } else { + if (input[9] > 156.50000000000003) { + var51 = -0.10803018621648303; + } else { + if (input[116] > 1e-35) { + var51 = -0.03208302566598311; + } else { + if (input[212] > 1e-35) { + if (input[243] > 1e-35) { + var51 = 0.10261721665006701; + } else { + var51 = 0.018994509090668264; + } + } else { + var51 = 0.0011244262442038839; + } + } + } + } + } + } + } + } + } + let var52: number; + if (input[86] > 1e-35) { + if (input[8] > 65.50000000000001) { + if (input[1] > 46.50000000000001) { + var52 = -0.08404263465005328; + } else { + if (input[0] > 3682.5000000000005) { + var52 = 0.041259223920298876; + } else { + if (input[1] > 29.500000000000004) { + var52 = -0.09541257493441671; + } else { + var52 = 0.001482192721625409; + } + } + } + } else { + var52 = 0.051541427372951004; + } + } else { + if (input[3] > 7.500000000000001) { + if (input[157] > 1e-35) { + var52 = -0.08268996098437432; + } else { + if (input[230] > 1e-35) { + var52 = 0.015749498159959817; + } else { + if (input[4] > 7.500000000000001) { + if (input[3] > 11.500000000000002) { + var52 = -0.0000913218977737457; + } else { + if (input[4] > 10.500000000000002) { + var52 = -0.056334165674005156; + } else { + if (input[127] > 1e-35) { + var52 = -0.0784634021824036; + } else { + if (input[2] > 9.500000000000002) { + if (input[1] > 62.50000000000001) { + var52 = -0.04231200150318989; + } else { + if (input[10] > 42.50000000000001) { + var52 = 0.10182973257894812; + } else { + var52 = 0.015934763950068445; + } + } + } else { + var52 = -0.03130938805859397; + } + } + } + } + } else { + if (input[92] > 1e-35) { + if (input[4] > 6.500000000000001) { + if (input[1] > 51.50000000000001) { + if (input[9] > 19.500000000000004) { + var52 = -0.041117068322885315; + } else { + var52 = 0.1167767830037126; + } + } else { + var52 = 0.13611206992387337; + } + } else { + if (input[10] > 41.50000000000001) { + var52 = -0.07120286010564107; + } else { + var52 = 0.022032788063345417; + } + } + } else { + if (input[8] > 1.5000000000000002) { + if (input[1] > 51.50000000000001) { + if (input[9] > 72.50000000000001) { + var52 = -0.07702290997669524; + } else { + if (input[198] > 1e-35) { + var52 = 0.08776558554437136; + } else { + var52 = -0.008290740324975692; + } + } + } else { + if (input[2] > 32.50000000000001) { + var52 = 0.07198457624219955; + } else { + var52 = 0.005463113714361629; + } + } + } else { + var52 = 0.09414099512900526; + } + } + } + } + } + } else { + if (input[129] > 1e-35) { + if (input[0] > 2904.5000000000005) { + if (input[0] > 4004.5000000000005) { + var52 = 0.03295785445437507; + } else { + var52 = 0.15140250150674536; + } + } else { + var52 = -0.035613213948910254; + } + } else { + if (input[186] > 1e-35) { + var52 = 0.06849425535860769; + } else { + if (input[96] > 1e-35) { + var52 = 0.06028225812727254; + } else { + var52 = -0.007582543288662308; + } + } + } + } + } + let var53: number; + if (input[84] > 1e-35) { + if (input[9] > 6.500000000000001) { + if (input[2] > 43.50000000000001) { + var53 = 0.08396556264106572; + } else { + var53 = -0.0562516995099192; + } + } else { + var53 = -0.10593011018789432; + } + } else { + if (input[183] > 1e-35) { + if (input[15] > 1e-35) { + var53 = -0.09705176473553752; + } else { + if (input[7] > 18.500000000000004) { + if (input[2] > 37.50000000000001) { + var53 = 0.0052017514017035915; + } else { + var53 = -0.11194119432743639; + } + } else { + var53 = 0.03724337696163019; + } + } + } else { + if (input[227] > 1e-35) { + if (input[17] > 1e-35) { + if (input[2] > 16.500000000000004) { + var53 = -0.025692451287403446; + } else { + var53 = -0.09511862672123193; + } + } else { + if (input[8] > 1661.5000000000002) { + if (input[10] > 37.50000000000001) { + var53 = -0.11892250746801664; + } else { + if (input[10] > 22.500000000000004) { + var53 = 0.07548493166973796; + } else { + var53 = -0.05973048107712209; + } + } + } else { + if (input[4] > 12.500000000000002) { + if (input[0] > 4319.500000000001) { + if (input[10] > 4.500000000000001) { + if (input[10] > 37.50000000000001) { + var53 = 0.13750699058082427; + } else { + if (input[18] > 1e-35) { + var53 = 0.06535408879552801; + } else { + var53 = -0.054118179035040674; + } + } + } else { + var53 = 0.1344282838979622; + } + } else { + if (input[0] > 3982.5000000000005) { + var53 = -0.10409582202467015; + } else { + if (input[19] > 1e-35) { + var53 = 0.12672850705810795; + } else { + if (input[8] > 587.5000000000001) { + if (input[1] > 35.50000000000001) { + var53 = 0.012705935670766466; + } else { + var53 = 0.14149359442527545; + } + } else { + var53 = -0.047977876173706004; + } + } + } + } + } else { + if (input[20] > 1e-35) { + var53 = 0.057945228080337946; + } else { + if (input[0] > 3642.5000000000005) { + var53 = -0.008726535792122467; + } else { + var53 = -0.08424769891378858; + } + } + } + } + } + } else { + if (input[34] > 1e-35) { + var53 = -0.0699329538228602; + } else { + if (input[134] > 1e-35) { + if (input[11] > 1e-35) { + if (input[4] > 15.500000000000002) { + if (input[0] > 1187.5000000000002) { + var53 = 0.01196849566739346; + } else { + var53 = 0.1614642278429876; + } + } else { + var53 = -0.043022338150701625; + } + } else { + if (input[3] > 5.500000000000001) { + var53 = -0.03907848255033881; + } else { + var53 = 0.018280601026175593; + } + } + } else { + var53 = 0.0006654540402589085; + } + } + } + } + } + let var54: number; + if (input[31] > 1e-35) { + if (input[2] > 58.50000000000001) { + if (input[9] > 1.5000000000000002) { + var54 = -0.01386103677247845; + } else { + var54 = 0.11386694333005128; + } + } else { + if (input[4] > 27.500000000000004) { + var54 = -0.021862617610091336; + } else { + if (input[2] > 31.500000000000004) { + var54 = 0.0828858469030438; + } else { + var54 = 0.006483353475830127; + } + } + } + } else { + if (input[224] > 1e-35) { + if (input[149] > 1e-35) { + if (input[13] > 1e-35) { + var54 = 0.11303635767048735; + } else { + var54 = -0.01645525128352694; + } + } else { + if (input[23] > 1e-35) { + if (input[4] > 62.50000000000001) { + var54 = -0.04238798044549342; + } else { + var54 = 0.022091190130494303; + } + } else { + if (input[5] > 5082.500000000001) { + var54 = -0.04287166152163786; + } else { + if (input[8] > 862.5000000000001) { + if (input[19] > 1e-35) { + var54 = 0.000660344696244351; + } else { + if (input[4] > 9.500000000000002) { + if (input[0] > 1277.5000000000002) { + var54 = -0.04291104140431434; + } else { + if (input[17] > 1e-35) { + var54 = 0.11256797532342613; + } else { + var54 = -0.017206916368289193; + } + } + } else { + var54 = 0.026482035265709743; + } + } + } else { + if (input[1] > 8.500000000000002) { + if (input[11] > 1e-35) { + var54 = 0.04060606971664621; + } else { + if (input[0] > 4733.500000000001) { + if (input[8] > 214.50000000000003) { + if (input[5] > 4814.500000000001) { + var54 = 0.03581712466863222; + } else { + var54 = 0.14770264307668884; + } + } else { + if (input[8] > 73.50000000000001) { + var54 = -0.13093289429740068; + } else { + var54 = 0.042461737442702936; + } + } + } else { + if (input[52] > 1e-35) { + var54 = 0.0501831919044939; + } else { + var54 = -0.010450249720465756; + } + } + } + } else { + var54 = -0.0753365425372656; + } + } + } + } + } + } else { + if (input[149] > 1e-35) { + if (input[6] > 23.500000000000004) { + var54 = 0.0005381332165438493; + } else { + var54 = -0.04549431717503909; + } + } else { + if (input[133] > 1e-35) { + if (input[2] > 5.500000000000001) { + if (input[8] > 698.5000000000001) { + if (input[282] > 1e-35) { + var54 = 0.04849637311285226; + } else { + var54 = -0.036671377119808564; + } + } else { + if (input[0] > 421.50000000000006) { + var54 = 0.00020968499911058945; + } else { + var54 = 0.11636422423182405; + } + } + } else { + var54 = -0.12687837788222575; + } + } else { + var54 = 0.0012774367867215346; + } + } + } + } + let var55: number; + if (input[120] > 1e-35) { + var55 = 0.04776057572434719; + } else { + if (input[229] > 1e-35) { + if (input[0] > 2952.5000000000005) { + if (input[0] > 3904.5000000000005) { + var55 = -0.042799574885345304; + } else { + var55 = 0.07412430171193245; + } + } else { + var55 = -0.11248270469336048; + } + } else { + if (input[193] > 1e-35) { + var55 = -0.060694220820603384; + } else { + if (input[121] > 1e-35) { + if (input[217] > 1e-35) { + if (input[0] > 4449.500000000001) { + if (input[4] > 8.500000000000002) { + var55 = 0.028911612178122104; + } else { + var55 = 0.12326369727728437; + } + } else { + if (input[0] > 4091.5000000000005) { + var55 = -0.09370267064141052; + } else { + if (input[0] > 3519.5000000000005) { + if (input[8] > 668.5000000000001) { + var55 = 0.1159839898100149; + } else { + var55 = -0.01924880886585737; + } + } else { + if (input[8] > 501.50000000000006) { + if (input[10] > 16.500000000000004) { + var55 = -0.0216343737351583; + } else { + var55 = -0.1220272260878369; + } + } else { + if (input[2] > 18.500000000000004) { + var55 = 0.09152924475072398; + } else { + if (input[8] > 55.50000000000001) { + var55 = 0.039508716651005665; + } else { + var55 = -0.11714436880423203; + } + } + } + } + } + } + } else { + if (input[18] > 1e-35) { + if (input[9] > 2.5000000000000004) { + var55 = 0.06793009902674053; + } else { + var55 = -0.024060578029812988; + } + } else { + if (input[4] > 2.5000000000000004) { + if (input[2] > 16.500000000000004) { + if (input[4] > 11.500000000000002) { + var55 = -0.04391068849624096; + } else { + var55 = 0.04009967593394672; + } + } else { + if (input[8] > 1085.5000000000002) { + var55 = -0.024773826356034825; + } else { + var55 = -0.13919707884246582; + } + } + } else { + var55 = 0.06659278075192335; + } + } + } + } else { + if (input[223] > 1e-35) { + if (input[8] > 668.5000000000001) { + var55 = -0.11567917501901476; + } else { + var55 = -0.006813640337684114; + } + } else { + if (input[3] > 7.500000000000001) { + var55 = 0.0010671269682548076; + } else { + if (input[7] > 3.5000000000000004) { + if (input[1] > 33.50000000000001) { + if (input[0] > 1597.5000000000002) { + if (input[10] > 1.5000000000000002) { + var55 = -0.001754586408351048; + } else { + var55 = -0.055422422450722056; + } + } else { + var55 = -0.06090032532532226; + } + } else { + if (input[0] > 5269.500000000001) { + var55 = 0.11787981735983527; + } else { + var55 = -0.00198119768540783; + } + } + } else { + var55 = 0.00210412924303036; + } + } + } + } + } + } + } + let var56: number; + if (input[294] > 1e-35) { + if (input[10] > 50.50000000000001) { + var56 = -0.09738558653332406; + } else { + if (input[0] > 2432.5000000000005) { + if (input[0] > 4533.500000000001) { + var56 = -0.06063239096209816; + } else { + var56 = 0.03317022411417386; + } + } else { + var56 = -0.08607562321324262; + } + } + } else { + if (input[120] > 1e-35) { + if (input[4] > 18.500000000000004) { + var56 = -0.013608609329298802; + } else { + var56 = 0.09078000157330264; + } + } else { + if (input[99] > 1e-35) { + var56 = 0.014828708581964632; + } else { + if (input[10] > 52.50000000000001) { + if (input[49] > 1e-35) { + var56 = -0.07536137260189814; + } else { + var56 = 0.006253266595455118; + } + } else { + if (input[10] > 28.500000000000004) { + var56 = -0.006106041147592768; + } else { + if (input[9] > 156.50000000000003) { + var56 = -0.11828932797811101; + } else { + if (input[94] > 1e-35) { + var56 = -0.02566078479505714; + } else { + if (input[303] > 1e-35) { + var56 = 0.09544850289775349; + } else { + if (input[15] > 1e-35) { + if (input[224] > 1e-35) { + if (input[4] > 56.50000000000001) { + var56 = -0.08401252789168523; + } else { + if (input[5] > 4244.500000000001) { + var56 = 0.026372887658499107; + } else { + if (input[1] > 16.500000000000004) { + var56 = -0.027836756345634026; + } else { + var56 = 0.09205362097909099; + } + } + } + } else { + var56 = 0.00934612788718244; + } + } else { + if (input[203] > 1e-35) { + var56 = -0.016371658366767253; + } else { + if (input[7] > 26.500000000000004) { + if (input[0] > 966.5000000000001) { + if (input[1] > 38.50000000000001) { + if (input[146] > 1e-35) { + if (input[9] > 21.500000000000004) { + var56 = -0.09580979052540028; + } else { + if (input[1] > 50.50000000000001) { + var56 = -0.06402211827281554; + } else { + var56 = 0.08342858760095972; + } + } + } else { + if (input[2] > 36.50000000000001) { + var56 = 0.008114897658204584; + } else { + if (input[92] > 1e-35) { + var56 = 0.09541587072672864; + } else { + var56 = -0.022342147210555434; + } + } + } + } else { + var56 = -0.01660492519175128; + } + } else { + var56 = 0.014721622240945446; + } + } else { + if (input[4] > 25.500000000000004) { + if (input[11] > 1e-35) { + var56 = 0.15846731118501817; + } else { + var56 = 0.039498507912023195; + } + } else { + if (input[245] > 1e-35) { + var56 = 0.07008718676813333; + } else { + var56 = 0.0019806389728814727; + } + } + } + } + } + } + } + } + } + } + } + } + } + let var57: number; + if (input[32] > 1e-35) { + if (input[8] > 90.50000000000001) { + if (input[4] > 67.50000000000001) { + if (input[0] > 4188.500000000001) { + var57 = -0.01192072916082109; + } else { + var57 = 0.13888590840802637; + } + } else { + if (input[1] > 16.500000000000004) { + if (input[8] > 2302.5000000000005) { + var57 = 0.06874032717466054; + } else { + if (input[4] > 40.50000000000001) { + var57 = -0.07752510020707537; + } else { + if (input[1] > 76.50000000000001) { + var57 = -0.09944032260703917; + } else { + if (input[8] > 1381.5000000000002) { + var57 = -0.054466635810800745; + } else { + if (input[1] > 32.50000000000001) { + var57 = 0.05974084520839573; + } else { + var57 = -0.0384718740755954; + } + } + } + } + } + } else { + var57 = -0.11374190719134032; + } + } + } else { + if (input[0] > 2151.5000000000005) { + var57 = -0.13703645155803298; + } else { + var57 = 0.004833344758654556; + } + } + } else { + if (input[297] > 1e-35) { + if (input[212] > 1e-35) { + var57 = 0.06954747264544993; + } else { + if (input[7] > 9.500000000000002) { + if (input[19] > 1e-35) { + if (input[1] > 30.500000000000004) { + if (input[0] > 4242.500000000001) { + var57 = 0.013539805885738608; + } else { + var57 = -0.0692740641801559; + } + } else { + if (input[0] > 2653.5000000000005) { + if (input[10] > 57.50000000000001) { + var57 = 0.09941880179344399; + } else { + var57 = -0.01608127391210995; + } + } else { + var57 = 0.08025226531247417; + } + } + } else { + if (input[9] > 67.50000000000001) { + var57 = 0.13525448212444113; + } else { + if (input[6] > 61.50000000000001) { + var57 = -0.05511099182158894; + } else { + if (input[94] > 1e-35) { + var57 = -0.06821509831783572; + } else { + if (input[128] > 1e-35) { + var57 = 0.11361314817714643; + } else { + var57 = 0.030160785008575566; + } + } + } + } + } + } else { + if (input[1] > 13.500000000000002) { + if (input[8] > 17.500000000000004) { + if (input[16] > 1e-35) { + var57 = -0.09954181329804547; + } else { + if (input[197] > 1e-35) { + var57 = 0.10102833149755386; + } else { + if (input[188] > 1e-35) { + var57 = 0.05584490988313965; + } else { + if (input[9] > 49.50000000000001) { + if (input[4] > 5.500000000000001) { + var57 = -0.03781554214742005; + } else { + var57 = 0.09927933385592314; + } + } else { + var57 = -0.020006000056720083; + } + } + } + } + } else { + var57 = -0.10520473615957895; + } + } else { + var57 = -0.12006990846253787; + } + } + } + } else { + var57 = -0.00026111570975317574; + } + } + let var58: number; + if (input[8] > 2830.5000000000005) { + if (input[1] > 31.500000000000004) { + if (input[9] > 32.50000000000001) { + if (input[5] > 1234.5000000000002) { + if (input[0] > 1725.5000000000002) { + if (input[7] > 14.500000000000002) { + if (input[2] > 38.50000000000001) { + var58 = -0.019188245509744628; + } else { + var58 = -0.13354864350075848; + } + } else { + if (input[0] > 2461.5000000000005) { + var58 = 0.051885477468354396; + } else { + var58 = -0.0833581968852119; + } + } + } else { + var58 = 0.08233441701532287; + } + } else { + var58 = -0.10865584951212362; + } + } else { + if (input[8] > 2992.5000000000005) { + if (input[10] > 49.50000000000001) { + if (input[10] > 56.50000000000001) { + if (input[1] > 45.50000000000001) { + if (input[0] > 2041.5000000000002) { + var58 = 0.09926337893072812; + } else { + var58 = -0.027753610497327715; + } + } else { + if (input[0] > 1972.5000000000002) { + var58 = -0.09780045823152517; + } else { + var58 = 0.032380915168504935; + } + } + } else { + var58 = 0.11502632261226381; + } + } else { + if (input[17] > 1e-35) { + var58 = -0.06094965899579662; + } else { + if (input[10] > 40.50000000000001) { + var58 = -0.07500475582440802; + } else { + var58 = 0.006499832113084677; + } + } + } + } else { + if (input[10] > 4.500000000000001) { + if (input[4] > 10.500000000000002) { + var58 = -0.09584538995220808; + } else { + var58 = -0.00908705814304442; + } + } else { + var58 = 0.03203281520813893; + } + } + } + } else { + if (input[10] > 49.50000000000001) { + var58 = -0.03146271513986384; + } else { + if (input[2] > 63.50000000000001) { + var58 = 0.13172001315536286; + } else { + if (input[224] > 1e-35) { + var58 = 0.08945777550527927; + } else { + if (input[0] > 2282.5000000000005) { + if (input[4] > 4.500000000000001) { + var58 = 0.09521549382082259; + } else { + var58 = -0.04414925613522197; + } + } else { + if (input[0] > 1847.5000000000002) { + var58 = -0.09118580379557353; + } else { + var58 = 0.009206744918282364; + } + } + } + } + } + } + } else { + if (input[178] > 1e-35) { + if (input[2] > 25.500000000000004) { + if (input[1] > 31.500000000000004) { + var58 = 0.03525144509943896; + } else { + var58 = -0.053340750721609057; + } + } else { + if (input[0] > 1057.5000000000002) { + if (input[10] > 2.5000000000000004) { + var58 = -0.04766112322938157; + } else { + if (input[2] > 10.500000000000002) { + var58 = 0.0728516504357201; + } else { + var58 = -0.05049625965272536; + } + } + } else { + var58 = -0.10868663055825774; + } + } + } else { + var58 = 0.0005382613419948969; + } + } + let var59: number; + if (input[147] > 1e-35) { + if (input[1] > 53.50000000000001) { + var59 = -0.10615739288764095; + } else { + if (input[0] > 2604.5000000000005) { + if (input[0] > 3629.5000000000005) { + var59 = -0.030504020655417463; + } else { + var59 = 0.07102458639110094; + } + } else { + var59 = -0.07058131985243714; + } + } + } else { + if (input[302] > 1e-35) { + if (input[10] > 47.50000000000001) { + var59 = 0.055304563442710876; + } else { + if (input[1] > 53.50000000000001) { + var59 = 0.033723409577443623; + } else { + if (input[8] > 175.50000000000003) { + if (input[0] > 2628.5000000000005) { + if (input[9] > 40.50000000000001) { + var59 = -0.1568835288372895; + } else { + var59 = -0.0279829124400056; + } + } else { + var59 = 0.04493843959601833; + } + } else { + var59 = -0.11637042729644327; + } + } + } + } else { + if (input[191] > 1e-35) { + if (input[282] > 1e-35) { + var59 = -0.054133834303687026; + } else { + if (input[9] > 48.50000000000001) { + var59 = 0.11263810289007213; + } else { + if (input[9] > 9.500000000000002) { + var59 = -0.02202034562838259; + } else { + if (input[4] > 45.50000000000001) { + var59 = -0.03410927569045158; + } else { + var59 = 0.04381615166534081; + } + } + } + } + } else { + if (input[242] > 1e-35) { + if (input[0] > 3615.5000000000005) { + if (input[3] > 19.500000000000004) { + if (input[1] > 56.50000000000001) { + if (input[4] > 28.500000000000004) { + var59 = -0.029687297407295893; + } else { + var59 = 0.10673602850001934; + } + } else { + if (input[4] > 42.50000000000001) { + var59 = 0.0036275562945108117; + } else { + var59 = -0.0760789221330622; + } + } + } else { + var59 = -0.10385623431741903; + } + } else { + if (input[2] > 34.50000000000001) { + if (input[2] > 44.50000000000001) { + if (input[4] > 51.50000000000001) { + var59 = 0.08274426793676076; + } else { + var59 = -0.07076234425516396; + } + } else { + var59 = 0.13890177606150175; + } + } else { + var59 = -0.019863286503635686; + } + } + } else { + if (input[53] > 1e-35) { + if (input[18] > 1e-35) { + var59 = -0.09250637750836187; + } else { + var59 = -0.0031531727902009026; + } + } else { + if (input[2] > 107.50000000000001) { + if (input[4] > 91.50000000000001) { + if (input[1] > 16.500000000000004) { + var59 = -0.01897867921812603; + } else { + var59 = 0.04890781705365262; + } + } else { + var59 = -0.11569892307597907; + } + } else { + if (input[2] > 106.50000000000001) { + var59 = 0.09032697440623969; + } else { + var59 = 0.00047935919155035045; + } + } + } + } + } + } + } + let var60: number; + if (input[115] > 1e-35) { + var60 = 0.05338335681275557; + } else { + if (input[242] > 1e-35) { + if (input[0] > 3615.5000000000005) { + if (input[4] > 42.50000000000001) { + if (input[4] > 75.50000000000001) { + var60 = -0.10131179514695865; + } else { + if (input[8] > 938.5000000000001) { + var60 = 0.10203729808015481; + } else { + var60 = -0.015357944186835289; + } + } + } else { + if (input[1] > 56.50000000000001) { + if (input[2] > 22.500000000000004) { + var60 = 0.03574015165562999; + } else { + var60 = -0.07763042506449493; + } + } else { + var60 = -0.0813323116215548; + } + } + } else { + if (input[2] > 34.50000000000001) { + if (input[2] > 44.50000000000001) { + if (input[4] > 51.50000000000001) { + var60 = 0.0665706259130275; + } else { + var60 = -0.06586817559309924; + } + } else { + var60 = 0.11925564412287476; + } + } else { + var60 = -0.014170019267143326; + } + } + } else { + if (input[1] > 124.50000000000001) { + if (input[2] > 30.500000000000004) { + if (input[8] > 533.5000000000001) { + if (input[4] > 41.50000000000001) { + if (input[8] > 977.5000000000001) { + var60 = 0.046017146627455346; + } else { + var60 = -0.08623321630086885; + } + } else { + if (input[8] > 1765.5000000000002) { + var60 = -0.017990564319859934; + } else { + if (input[10] > 25.500000000000004) { + if (input[10] > 48.50000000000001) { + var60 = 0.11143827902215087; + } else { + var60 = -0.01817808730473413; + } + } else { + var60 = 0.16980985030210127; + } + } + } + } else { + var60 = -0.09357806298740017; + } + } else { + if (input[10] > 7.500000000000001) { + if (input[10] > 54.50000000000001) { + var60 = 0.010168994879727824; + } else { + var60 = -0.09099594488792513; + } + } else { + if (input[9] > 1.5000000000000002) { + var60 = 0.0533459678147928; + } else { + var60 = -0.06886854808370108; + } + } + } + } else { + if (input[99] > 1e-35) { + if (input[17] > 1e-35) { + if (input[9] > 22.500000000000004) { + var60 = -0.062346959148773695; + } else { + if (input[1] > 47.50000000000001) { + var60 = -0.0021578343835599316; + } else { + if (input[2] > 27.500000000000004) { + var60 = 0.19567373210166172; + } else { + var60 = 0.07851555379116423; + } + } + } + } else { + if (input[18] > 1e-35) { + var60 = 0.03711549097804649; + } else { + if (input[8] > 359.50000000000006) { + var60 = 0.012492346746905587; + } else { + if (input[4] > 20.500000000000004) { + var60 = 0.047511695735697544; + } else { + var60 = -0.07999269063948773; + } + } + } + } + } else { + var60 = 0.00006802045404471004; + } + } + } + } + let var61: number; + if (input[222] > 1e-35) { + if (input[0] > 612.5000000000001) { + if (input[10] > 1e-35) { + if (input[8] > 2167.5000000000005) { + if (input[4] > 25.500000000000004) { + var61 = 0.0011484728213539738; + } else { + var61 = -0.0936582904650763; + } + } else { + if (input[2] > 25.500000000000004) { + if (input[8] > 182.50000000000003) { + if (input[10] > 22.500000000000004) { + if (input[0] > 5026.500000000001) { + var61 = -0.09828874964938798; + } else { + if (input[8] > 1586.5000000000002) { + var61 = 0.13726397438080162; + } else { + if (input[4] > 48.50000000000001) { + if (input[2] > 63.50000000000001) { + var61 = 0.011938269926919522; + } else { + var61 = 0.17541983715953954; + } + } else { + if (input[19] > 1e-35) { + var61 = 0.023002786011088672; + } else { + var61 = -0.06221461272461431; + } + } + } + } + } else { + if (input[9] > 2.5000000000000004) { + if (input[0] > 3818.5000000000005) { + var61 = 0.06508934844183291; + } else { + var61 = -0.10168553534835639; + } + } else { + var61 = -0.07755626499024171; + } + } + } else { + if (input[2] > 51.50000000000001) { + if (input[4] > 65.50000000000001) { + var61 = 0.021140806225203937; + } else { + var61 = -0.1167833342453639; + } + } else { + if (input[2] > 33.50000000000001) { + var61 = 0.13163585734056618; + } else { + var61 = -0.00203273890889717; + } + } + } + } else { + if (input[10] > 36.50000000000001) { + if (input[8] > 1067.5000000000002) { + var61 = 0.06314479201263888; + } else { + var61 = -0.09639088327091713; + } + } else { + if (input[10] > 29.500000000000004) { + var61 = 0.09225469303582386; + } else { + if (input[0] > 3129.5000000000005) { + if (input[0] > 4091.5000000000005) { + if (input[0] > 4354.500000000001) { + var61 = 0.000040577156464836036; + } else { + var61 = 0.12322387121810757; + } + } else { + var61 = -0.03697224045046014; + } + } else { + if (input[1] > 22.500000000000004) { + var61 = 0.016474835887320276; + } else { + var61 = 0.16919298733903063; + } + } + } + } + } + } + } else { + var61 = 0.07633203630214054; + } + } else { + var61 = -0.047438037934250644; + } + } else { + if (input[30] > 1e-35) { + if (input[224] > 1e-35) { + if (input[1] > 52.50000000000001) { + var61 = 0.14150493354700563; + } else { + var61 = -0.01831155354975749; + } + } else { + if (input[1] > 28.500000000000004) { + var61 = -0.07952557178685365; + } else { + if (input[10] > 28.500000000000004) { + var61 = 0.0665695554984927; + } else { + var61 = -0.053640139319277094; + } + } + } + } else { + var61 = 0.0004754840665898665; + } + } + let var62: number; + if (input[76] > 1e-35) { + var62 = -0.06814884255939921; + } else { + if (input[179] > 1e-35) { + var62 = -0.06325743795510681; + } else { + if (input[122] > 1e-35) { + if (input[6] > 36.50000000000001) { + var62 = 0.05052338063261613; + } else { + if (input[8] > 626.5000000000001) { + if (input[1] > 38.50000000000001) { + var62 = 0.004193658608848433; + } else { + var62 = -0.1066968975983452; + } + } else { + if (input[8] > 302.50000000000006) { + var62 = 0.05476730110440451; + } else { + var62 = -0.06382970920394895; + } + } + } + } else { + if (input[218] > 1e-35) { + if (input[2] > 3.5000000000000004) { + if (input[6] > 13.500000000000002) { + if (input[2] > 19.500000000000004) { + if (input[0] > 3200.5000000000005) { + if (input[4] > 91.50000000000001) { + var62 = -0.12156071809840739; + } else { + if (input[9] > 21.500000000000004) { + if (input[5] > 3883.5000000000005) { + if (input[8] > 919.5000000000001) { + if (input[8] > 1085.5000000000002) { + var62 = 0.013555772109446666; + } else { + var62 = -0.09856116699770784; + } + } else { + var62 = 0.0284329611813383; + } + } else { + if (input[2] > 52.50000000000001) { + var62 = 0.04008708444763762; + } else { + if (input[9] > 29.500000000000004) { + var62 = -0.1289599546008197; + } else { + var62 = -0.018566534248335896; + } + } + } + } else { + if (input[8] > 747.5000000000001) { + var62 = 0.02236484980076122; + } else { + var62 = 0.1148871655157582; + } + } + } + } else { + if (input[8] > 3084.0000000000005) { + var62 = -0.05573875952902531; + } else { + if (input[10] > 17.500000000000004) { + if (input[2] > 51.50000000000001) { + var62 = 0.03164751204281298; + } else { + var62 = 0.11752140436184891; + } + } else { + if (input[9] > 42.50000000000001) { + var62 = -0.07180559595410106; + } else { + if (input[22] > 1e-35) { + var62 = 0.09325040416256854; + } else { + var62 = -0.016041122807939914; + } + } + } + } + } + } else { + var62 = -0.02765708954618808; + } + } else { + if (input[1] > 30.500000000000004) { + if (input[1] > 66.50000000000001) { + var62 = -0.010718250133458515; + } else { + var62 = 0.09818827994853763; + } + } else { + var62 = 0.010180038981174032; + } + } + } else { + var62 = -0.039472162599295535; + } + } else { + if (input[9] > 170.50000000000003) { + var62 = -0.08536729235976731; + } else { + if (input[189] > 1e-35) { + if (input[0] > 5269.500000000001) { + var62 = -0.08674788057474031; + } else { + var62 = 0.02077653508548371; + } + } else { + var62 = -0.0003536561382007414; + } + } + } + } + } + } + let var63: number; + if (input[86] > 1e-35) { + if (input[10] > 6.500000000000001) { + if (input[0] > 4376.500000000001) { + var63 = 0.018337297491457794; + } else { + var63 = -0.05926206443180149; + } + } else { + var63 = 0.024026520855881126; + } + } else { + if (input[288] > 1e-35) { + if (input[184] > 1e-35) { + var63 = 0.10747078482128616; + } else { + if (input[126] > 1e-35) { + var63 = -0.10550625192391357; + } else { + if (input[7] > 71.50000000000001) { + var63 = -0.07698346027863572; + } else { + if (input[8] > 302.50000000000006) { + if (input[6] > 49.50000000000001) { + if (input[4] > 47.50000000000001) { + if (input[1] > 38.50000000000001) { + if (input[15] > 1e-35) { + var63 = 0.1317396472229434; + } else { + var63 = -0.025035791351328947; + } + } else { + var63 = -0.0728334305864372; + } + } else { + if (input[8] > 963.5000000000001) { + var63 = 0.023642201723096064; + } else { + var63 = 0.183010326734258; + } + } + } else { + if (input[128] > 1e-35) { + var63 = 0.04228920135648387; + } else { + if (input[2] > 34.50000000000001) { + if (input[15] > 1e-35) { + var63 = 0.002801782941492993; + } else { + if (input[3] > 40.50000000000001) { + if (input[4] > 39.50000000000001) { + var63 = -0.1088876900335281; + } else { + var63 = 0.02758317023002635; + } + } else { + var63 = -0.11886771300807207; + } + } + } else { + if (input[9] > 59.50000000000001) { + if (input[1] > 33.50000000000001) { + var63 = -0.01928020117446408; + } else { + var63 = 0.10193718474139135; + } + } else { + if (input[1] > 48.50000000000001) { + if (input[4] > 9.500000000000002) { + if (input[8] > 932.5000000000001) { + var63 = 0.07893723375925096; + } else { + var63 = -0.009878929627026153; + } + } else { + if (input[10] > 2.5000000000000004) { + if (input[9] > 20.500000000000004) { + var63 = -0.10301657587280551; + } else { + var63 = 0.005787463140224318; + } + } else { + var63 = 0.07421364314695046; + } + } + } else { + if (input[0] > 2840.5000000000005) { + if (input[10] > 29.500000000000004) { + var63 = -0.019296977889522397; + } else { + var63 = -0.07274529751752634; + } + } else { + if (input[1] > 30.500000000000004) { + var63 = -0.050368901143148286; + } else { + var63 = 0.029630869489466655; + } + } + } + } + } + } + } + } else { + if (input[2] > 6.500000000000001) { + if (input[4] > 9.500000000000002) { + var63 = 0.0015332402792773946; + } else { + var63 = 0.09930153676749967; + } + } else { + var63 = -0.06370844564357069; + } + } + } + } + } + } else { + var63 = 0.00042272155209927616; + } + } + let var64: number; + if (input[71] > 1e-35) { + if (input[4] > 17.500000000000004) { + var64 = 0.12586844370423247; + } else { + var64 = -0.006791999603126354; + } + } else { + if (input[222] > 1e-35) { + if (input[1] > 10.500000000000002) { + if (input[30] > 1e-35) { + if (input[1] > 36.50000000000001) { + if (input[9] > 1.5000000000000002) { + if (input[10] > 25.500000000000004) { + var64 = -0.08474891624263797; + } else { + if (input[8] > 125.50000000000001) { + var64 = 0.08125086980439704; + } else { + var64 = -0.04082085238068532; + } + } + } else { + if (input[0] > 3863.5000000000005) { + var64 = 0.020481535807469208; + } else { + var64 = 0.14810819386202126; + } + } + } else { + if (input[0] > 1937.5000000000002) { + if (input[2] > 16.500000000000004) { + var64 = -0.019110200161573936; + } else { + var64 = -0.12387719685855114; + } + } else { + if (input[0] > 1102.5000000000002) { + var64 = 0.08376595701957407; + } else { + var64 = -0.031821919580524834; + } + } + } + } else { + if (input[9] > 4.500000000000001) { + var64 = -0.08116383486497568; + } else { + if (input[7] > 8.500000000000002) { + if (input[2] > 24.500000000000004) { + var64 = -0.02154820850475448; + } else { + if (input[0] > 3863.5000000000005) { + if (input[8] > 902.5000000000001) { + var64 = 0.1349841206807871; + } else { + var64 = 0.011864053595560297; + } + } else { + if (input[1] > 41.50000000000001) { + var64 = -0.08203662486612544; + } else { + if (input[2] > 18.500000000000004) { + var64 = -0.009541865642346947; + } else { + var64 = 0.08345043168501759; + } + } + } + } + } else { + if (input[2] > 10.500000000000002) { + var64 = -0.09585031818030947; + } else { + var64 = 0.019432330487099865; + } + } + } + } + } else { + var64 = 0.08399259524715129; + } + } else { + if (input[30] > 1e-35) { + if (input[224] > 1e-35) { + if (input[1] > 52.50000000000001) { + var64 = 0.11951517733981365; + } else { + var64 = -0.016651014735738538; + } + } else { + if (input[1] > 28.500000000000004) { + var64 = -0.07410922545030711; + } else { + if (input[10] > 28.500000000000004) { + var64 = 0.05886430683844788; + } else { + var64 = -0.04929626605117184; + } + } + } + } else { + if (input[191] > 1e-35) { + if (input[9] > 9.500000000000002) { + if (input[9] > 48.50000000000001) { + var64 = 0.04802269879144705; + } else { + var64 = -0.026208212831796737; + } + } else { + if (input[4] > 45.50000000000001) { + var64 = -0.03227476944664786; + } else { + var64 = 0.05124575625622705; + } + } + } else { + var64 = 0.00020506696916003137; + } + } + } + } + let var65: number; + if (input[116] > 1e-35) { + if (input[9] > 2.5000000000000004) { + if (input[9] > 17.500000000000004) { + var65 = -0.03042091758483443; + } else { + if (input[10] > 14.500000000000002) { + var65 = 0.09816619204768777; + } else { + var65 = 0.01332124067720947; + } + } + } else { + if (input[8] > 8.500000000000002) { + if (input[4] > 15.500000000000002) { + var65 = -0.02381165060401718; + } else { + var65 = -0.10950361804974783; + } + } else { + var65 = 0.03538211665111128; + } + } + } else { + if (input[212] > 1e-35) { + if (input[19] > 1e-35) { + var65 = -0.09940014650006174; + } else { + if (input[0] > 2215.5000000000005) { + if (input[5] > 5056.500000000001) { + if (input[3] > 5.500000000000001) { + if (input[10] > 25.500000000000004) { + var65 = -0.06371052144380579; + } else { + var65 = 0.0835500621252692; + } + } else { + var65 = -0.10408255929333915; + } + } else { + if (input[1] > 74.50000000000001) { + var65 = 0.13208968122712403; + } else { + if (input[1] > 64.50000000000001) { + var65 = -0.04778844603644965; + } else { + if (input[8] > 51.50000000000001) { + if (input[8] > 201.50000000000003) { + if (input[8] > 660.5000000000001) { + if (input[6] > 4.500000000000001) { + if (input[9] > 5.500000000000001) { + if (input[1] > 29.500000000000004) { + if (input[0] > 3830.5000000000005) { + var65 = 0.09922816902423433; + } else { + var65 = 0.016366955328796718; + } + } else { + var65 = 0.1592412560903584; + } + } else { + if (input[1] > 39.50000000000001) { + var65 = 0.05409467990258923; + } else { + var65 = -0.08260633210459611; + } + } + } else { + var65 = -0.06307205775247567; + } + } else { + if (input[9] > 36.50000000000001) { + var65 = 0.040253940015648144; + } else { + var65 = 0.14202568969471283; + } + } + } else { + var65 = -0.028761848341594044; + } + } else { + var65 = 0.08994073058773508; + } + } + } + } + } else { + if (input[0] > 807.5000000000001) { + var65 = -0.043427848826323195; + } else { + var65 = 0.04573516446846493; + } + } + } + } else { + if (input[20] > 1e-35) { + if (input[188] > 1e-35) { + var65 = -0.0758877731600639; + } else { + if (input[23] > 1e-35) { + var65 = 0.05913923322043199; + } else { + if (input[8] > 155.50000000000003) { + if (input[128] > 1e-35) { + var65 = 0.08124700978741987; + } else { + var65 = 0.013296063087086852; + } + } else { + if (input[7] > 5.500000000000001) { + var65 = -0.01640196088612987; + } else { + var65 = -0.12685498840146067; + } + } + } + } + } else { + var65 = -0.0004940792382459551; + } + } + } + let var66: number; + if (input[1] > 24.500000000000004) { + if (input[103] > 1e-35) { + if (input[8] > 61.50000000000001) { + if (input[17] > 1e-35) { + var66 = -0.05584993681929434; + } else { + if (input[9] > 27.500000000000004) { + if (input[0] > 3916.5000000000005) { + var66 = 0.08513773825688947; + } else { + var66 = -0.1184664832315282; + } + } else { + var66 = 0.05676963535893477; + } + } + } else { + var66 = 0.14263843210340613; + } + } else { + var66 = 0.0005795003292924202; + } + } else { + if (input[18] > 1e-35) { + if (input[0] > 5453.500000000001) { + if (input[1] > 11.500000000000002) { + var66 = -0.10669720555606924; + } else { + var66 = 0.029016613003137307; + } + } else { + if (input[2] > 46.50000000000001) { + if (input[10] > 9.500000000000002) { + var66 = 0.0664744575868955; + } else { + var66 = -0.08469256188890871; + } + } else { + var66 = -0.026746678040592144; + } + } + } else { + if (input[281] > 1e-35) { + var66 = -0.07408427239006925; + } else { + if (input[145] > 1e-35) { + if (input[4] > 6.500000000000001) { + if (input[9] > 16.500000000000004) { + if (input[4] > 18.500000000000004) { + var66 = 0.012131807587207655; + } else { + var66 = -0.12776015795398743; + } + } else { + var66 = 0.04320472481083551; + } + } else { + var66 = 0.08390980661550446; + } + } else { + if (input[10] > 227.50000000000003) { + var66 = -0.09771783809101153; + } else { + if (input[10] > 130.50000000000003) { + var66 = 0.11175201938704937; + } else { + if (input[8] > 779.5000000000001) { + if (input[5] > 3325.5000000000005) { + if (input[128] > 1e-35) { + var66 = -0.07610698254064358; + } else { + if (input[8] > 902.5000000000001) { + var66 = -0.03136381213599649; + } else { + if (input[131] > 1e-35) { + var66 = 0.0704821739127936; + } else { + if (input[224] > 1e-35) { + var66 = -0.056961477774953785; + } else { + if (input[10] > 30.500000000000004) { + if (input[9] > 43.50000000000001) { + var66 = 0.10431473040024908; + } else { + if (input[8] > 841.5000000000001) { + var66 = 0.07304745320500514; + } else { + var66 = -0.038011541882439825; + } + } + } else { + var66 = -0.01679746695007364; + } + } + } + } + } + } else { + if (input[0] > 3129.5000000000005) { + var66 = 0.05589952587431965; + } else { + if (input[210] > 1e-35) { + var66 = 0.06227198085800842; + } else { + var66 = -0.0011341890997947812; + } + } + } + } else { + if (input[8] > 740.5000000000001) { + var66 = 0.04817300084412584; + } else { + var66 = -0.000577001010789238; + } + } + } + } + } + } + } + } + let var67: number; + if (input[187] > 1e-35) { + if (input[6] > 12.500000000000002) { + if (input[10] > 8.500000000000002) { + if (input[10] > 16.500000000000004) { + if (input[8] > 234.50000000000003) { + if (input[4] > 43.50000000000001) { + if (input[0] > 4476.500000000001) { + var67 = -0.10504730480402079; + } else { + if (input[5] > 3341.5000000000005) { + var67 = 0.11087894671081754; + } else { + var67 = -0.0406668834674614; + } + } + } else { + var67 = 0.03308382165616109; + } + } else { + if (input[8] > 104.50000000000001) { + var67 = -0.10431436764549162; + } else { + var67 = 0.0073928337244891455; + } + } + } else { + if (input[4] > 34.50000000000001) { + var67 = -0.10571751512748416; + } else { + var67 = -0.006081128814142983; + } + } + } else { + if (input[13] > 1e-35) { + var67 = 0.1299673566095023; + } else { + if (input[4] > 60.50000000000001) { + var67 = -0.06587492443829139; + } else { + if (input[0] > 2604.5000000000005) { + if (input[3] > 19.500000000000004) { + var67 = 0.04857126072645073; + } else { + var67 = -0.03431365358104773; + } + } else { + if (input[4] > 16.500000000000004) { + var67 = 0.04101865986596709; + } else { + var67 = 0.16480274980378218; + } + } + } + } + } + } else { + if (input[10] > 26.500000000000004) { + var67 = 0.03673978504199255; + } else { + if (input[10] > 9.500000000000002) { + var67 = -0.10996402743800027; + } else { + if (input[308] > 1e-35) { + var67 = 0.0553693735082498; + } else { + var67 = -0.041600136235644125; + } + } + } + } + } else { + if (input[306] > 1e-35) { + if (input[8] > 1156.5000000000002) { + if (input[4] > 14.500000000000002) { + if (input[10] > 21.500000000000004) { + var67 = 0.010902983761213922; + } else { + var67 = 0.1325118659895645; + } + } else { + var67 = -0.064362945508595; + } + } else { + if (input[1] > 66.50000000000001) { + var67 = 0.033416767779331176; + } else { + var67 = -0.054080316225040496; + } + } + } else { + if (input[42] > 1e-35) { + var67 = -0.07762364337810815; + } else { + if (input[10] > 1089.5000000000002) { + var67 = -0.08465599849125216; + } else { + if (input[31] > 1e-35) { + if (input[8] > 30.500000000000004) { + var67 = 0.012788520036013586; + } else { + if (input[1] > 32.50000000000001) { + if (input[1] > 51.50000000000001) { + var67 = 0.0220102041325908; + } else { + var67 = -0.06516708740003069; + } + } else { + var67 = 0.012833498905748267; + } + } + } else { + if (input[224] > 1e-35) { + var67 = -0.007038418272997865; + } else { + var67 = 0.00037666304316290967; + } + } + } + } + } + } + let var68: number; + if (input[84] > 1e-35) { + if (input[9] > 6.500000000000001) { + if (input[2] > 43.50000000000001) { + var68 = 0.07554189644995735; + } else { + var68 = -0.052089349455904946; + } + } else { + var68 = -0.10148206848169845; + } + } else { + if (input[113] > 1e-35) { + var68 = -0.06666678653225779; + } else { + if (input[39] > 1e-35) { + if (input[9] > 3.5000000000000004) { + if (input[0] > 3670.5000000000005) { + var68 = 0.07172653627995676; + } else { + var68 = -0.07602959317610998; + } + } else { + var68 = -0.08790686271287523; + } + } else { + if (input[229] > 1e-35) { + if (input[0] > 2952.5000000000005) { + if (input[0] > 3904.5000000000005) { + var68 = -0.0399322883690891; + } else { + var68 = 0.06523495517476098; + } + } else { + var68 = -0.10358715295743802; + } + } else { + if (input[193] > 1e-35) { + var68 = -0.05551414334329124; + } else { + if (input[134] > 1e-35) { + if (input[11] > 1e-35) { + if (input[2] > 13.500000000000002) { + if (input[10] > 1.5000000000000002) { + var68 = 0.015928764772252406; + } else { + var68 = 0.1341513061552287; + } + } else { + var68 = -0.04975001987586173; + } + } else { + if (input[10] > 2.5000000000000004) { + if (input[3] > 5.500000000000001) { + if (input[9] > 2.5000000000000004) { + if (input[8] > 310.50000000000006) { + var68 = -0.033592997607280156; + } else { + var68 = -0.12432458028446665; + } + } else { + if (input[1] > 32.50000000000001) { + if (input[217] > 1e-35) { + var68 = -0.08402551858097379; + } else { + var68 = 0.017401984506038796; + } + } else { + if (input[1] > 25.500000000000004) { + var68 = 0.13337205393591278; + } else { + var68 = -0.01160208350090984; + } + } + } + } else { + var68 = 0.06708317942315471; + } + } else { + if (input[8] > 227.50000000000003) { + var68 = -0.08486943882418681; + } else { + var68 = -0.013970104864235007; + } + } + } + } else { + if (input[8] > 4968.500000000001) { + if (input[1] > 31.500000000000004) { + if (input[9] > 4.500000000000001) { + var68 = -0.10496268177586783; + } else { + var68 = -0.020921489532370493; + } + } else { + var68 = 0.02629915927247642; + } + } else { + if (input[7] > 20.500000000000004) { + if (input[8] > 251.50000000000003) { + if (input[115] > 1e-35) { + var68 = 0.11639296062157028; + } else { + var68 = -0.004275784356569115; + } + } else { + if (input[32] > 1e-35) { + var68 = -0.07297384970166025; + } else { + var68 = 0.006026841626381599; + } + } + } else { + var68 = 0.002034611134960428; + } + } + } + } + } + } + } + } + let var69: number; + if (input[248] > 1e-35) { + var69 = 0.06091438745093315; + } else { + if (input[0] > 384.50000000000006) { + if (input[204] > 1e-35) { + if (input[1] > 62.50000000000001) { + var69 = -0.06455513326540585; + } else { + if (input[1] > 29.500000000000004) { + var69 = 0.07718474591552532; + } else { + if (input[4] > 7.500000000000001) { + var69 = 0.040139336931404826; + } else { + var69 = -0.09685734690563386; + } + } + } + } else { + var69 = 0.00015327283570347363; + } + } else { + if (input[9] > 88.50000000000001) { + var69 = 0.10079017954199324; + } else { + if (input[1] > 47.50000000000001) { + if (input[2] > 20.500000000000004) { + if (input[2] > 27.500000000000004) { + var69 = -0.04077257804338707; + } else { + var69 = 0.0739963982640615; + } + } else { + if (input[9] > 1.5000000000000002) { + if (input[17] > 1e-35) { + var69 = 0.03778141591008941; + } else { + var69 = -0.06459919920634845; + } + } else { + var69 = -0.11193190957880604; + } + } + } else { + if (input[7] > 6.500000000000001) { + if (input[11] > 1e-35) { + if (input[18] > 1e-35) { + var69 = 0.14063930759326346; + } else { + if (input[0] > 179.50000000000003) { + var69 = 0.07287482250668585; + } else { + if (input[8] > 1180.5000000000002) { + var69 = -0.14419393112726253; + } else { + if (input[10] > 28.500000000000004) { + var69 = -0.07993142770099469; + } else { + if (input[17] > 1e-35) { + var69 = -0.04702595410391655; + } else { + if (input[7] > 21.500000000000004) { + if (input[2] > 26.500000000000004) { + var69 = 0.05527969663610186; + } else { + var69 = -0.10824385941441346; + } + } else { + if (input[3] > 11.500000000000002) { + var69 = 0.12358502961047915; + } else { + var69 = -0.017509147119622873; + } + } + } + } + } + } + } + } else { + if (input[0] > 74.50000000000001) { + var69 = -0.014907705458730486; + } else { + if (input[8] > 95.50000000000001) { + var69 = -0.02225118168342062; + } else { + var69 = -0.1222374623708485; + } + } + } + } else { + if (input[8] > 1.5000000000000002) { + if (input[8] > 950.5000000000001) { + var69 = 0.06946188930925638; + } else { + if (input[3] > 6.500000000000001) { + if (input[10] > 2.5000000000000004) { + if (input[19] > 1e-35) { + var69 = 0.04962819555610421; + } else { + var69 = -0.07213577821855309; + } + } else { + var69 = 0.09139529824708481; + } + } else { + if (input[19] > 1e-35) { + var69 = 0.013439401088345224; + } else { + var69 = -0.049274647207292056; + } + } + } + } else { + var69 = 0.10531673719686951; + } + } + } + } + } + } + let var70: number; + if (input[40] > 1e-35) { + if (input[0] > 1937.5000000000002) { + var70 = -0.06421671152073961; + } else { + var70 = 0.04235421241226177; + } + } else { + if (input[294] > 1e-35) { + if (input[10] > 50.50000000000001) { + var70 = -0.09100102290316286; + } else { + if (input[0] > 3030.5000000000005) { + if (input[0] > 4177.500000000001) { + var70 = -0.03520420769287065; + } else { + if (input[8] > 1085.5000000000002) { + var70 = -0.019817352506127633; + } else { + var70 = 0.11444439424520964; + } + } + } else { + var70 = -0.06854631664538167; + } + } + } else { + if (input[120] > 1e-35) { + if (input[4] > 18.500000000000004) { + var70 = -0.010490117519863269; + } else { + var70 = 0.08104430117757461; + } + } else { + if (input[121] > 1e-35) { + if (input[243] > 1e-35) { + var70 = 0.16408304891242204; + } else { + if (input[217] > 1e-35) { + if (input[0] > 4449.500000000001) { + var70 = 0.06619344145920268; + } else { + if (input[0] > 4091.5000000000005) { + var70 = -0.08813353450871053; + } else { + if (input[0] > 3519.5000000000005) { + if (input[8] > 668.5000000000001) { + var70 = 0.10016091391222309; + } else { + var70 = -0.017407607199427293; + } + } else { + if (input[8] > 501.50000000000006) { + if (input[10] > 16.500000000000004) { + var70 = -0.019511460451434884; + } else { + var70 = -0.11643672465055221; + } + } else { + if (input[2] > 18.500000000000004) { + var70 = 0.07848228087333317; + } else { + if (input[8] > 55.50000000000001) { + var70 = 0.032583027899956235; + } else { + var70 = -0.11209832692153521; + } + } + } + } + } + } + } else { + if (input[11] > 1e-35) { + var70 = 0.027482174104412567; + } else { + if (input[10] > 1.5000000000000002) { + if (input[6] > 26.500000000000004) { + if (input[4] > 19.500000000000004) { + if (input[9] > 31.500000000000004) { + var70 = -0.09996887746328006; + } else { + if (input[9] > 2.5000000000000004) { + var70 = 0.02157682011863397; + } else { + var70 = -0.05247727848991843; + } + } + } else { + var70 = 0.07409150201483244; + } + } else { + if (input[1] > 38.50000000000001) { + var70 = -0.11378466075449625; + } else { + if (input[224] > 1e-35) { + var70 = -0.10741749127732923; + } else { + if (input[1] > 26.500000000000004) { + var70 = 0.07343136534146562; + } else { + var70 = -0.07013573628594773; + } + } + } + } + } else { + if (input[25] > 1e-35) { + var70 = -0.04626669734164317; + } else { + var70 = 0.05518333197956482; + } + } + } + } + } + } else { + var70 = 0.00032434010867555516; + } + } + } + } + let var71: number; + if (input[183] > 1e-35) { + if (input[10] > 1.5000000000000002) { + if (input[17] > 1e-35) { + var71 = 0.026313251010808853; + } else { + var71 = -0.08997339150292381; + } + } else { + var71 = 0.025062509535227952; + } + } else { + if (input[227] > 1e-35) { + if (input[1] > 6.500000000000001) { + if (input[2] > 9.500000000000002) { + if (input[210] > 1e-35) { + var71 = 0.08071107515789745; + } else { + if (input[23] > 1e-35) { + if (input[1] > 75.50000000000001) { + var71 = 0.0905155504503746; + } else { + if (input[8] > 1049.5000000000002) { + var71 = -0.062312558183394054; + } else { + if (input[8] > 719.5000000000001) { + var71 = 0.09583836191410239; + } else { + if (input[0] > 3719.5000000000005) { + var71 = -0.0778097309430818; + } else { + var71 = 0.04012012419054895; + } + } + } + } + } else { + if (input[4] > 12.500000000000002) { + if (input[8] > 1496.5000000000002) { + if (input[10] > 42.50000000000001) { + var71 = -0.12920865648544927; + } else { + if (input[0] > 2699.5000000000005) { + var71 = -0.07086587879041864; + } else { + var71 = 0.022614182502461846; + } + } + } else { + if (input[4] > 15.500000000000002) { + if (input[8] > 55.50000000000001) { + if (input[1] > 60.50000000000001) { + if (input[8] > 652.5000000000001) { + var71 = -0.11377786322600797; + } else { + var71 = -0.009486325820117998; + } + } else { + if (input[1] > 55.50000000000001) { + var71 = 0.12430248795958142; + } else { + if (input[0] > 2952.5000000000005) { + if (input[0] > 4331.500000000001) { + if (input[1] > 38.50000000000001) { + var71 = -0.07938291201004219; + } else { + if (input[2] > 36.50000000000001) { + var71 = 0.01520046732530246; + } else { + var71 = 0.13649854049662832; + } + } + } else { + var71 = -0.07145015938528873; + } + } else { + if (input[8] > 407.50000000000006) { + var71 = -0.00350257360822279; + } else { + var71 = 0.11332047082193297; + } + } + } + } + } else { + var71 = -0.10060624458629897; + } + } else { + var71 = 0.05429496612497562; + } + } + } else { + if (input[8] > 1446.5000000000002) { + var71 = 0.006073419197482838; + } else { + var71 = -0.08718676350883998; + } + } + } + } + } else { + var71 = -0.11532497988252638; + } + } else { + var71 = 0.10766270463068293; + } + } else { + if (input[34] > 1e-35) { + var71 = -0.06345912440611544; + } else { + if (input[131] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var71 = -0.0004109812623829506; + } else { + var71 = 0.021601073497455662; + } + } else { + var71 = -0.00007343540098965853; + } + } + } + } + let var72: number; + if (input[298] > 1e-35) { + if (input[9] > 12.500000000000002) { + if (input[133] > 1e-35) { + var72 = -0.06107663265515864; + } else { + if (input[9] > 70.50000000000001) { + if (input[10] > 37.50000000000001) { + var72 = 0.05995640200798119; + } else { + if (input[0] > 3443.5000000000005) { + var72 = -0.14698883458733583; + } else { + var72 = -0.030039164579240187; + } + } + } else { + if (input[189] > 1e-35) { + var72 = -0.06086763220538141; + } else { + if (input[1] > 86.50000000000001) { + var72 = -0.05096727866142538; + } else { + if (input[4] > 64.50000000000001) { + var72 = 0.11240554253834577; + } else { + if (input[4] > 45.50000000000001) { + var72 = -0.030279760168394117; + } else { + if (input[6] > 45.50000000000001) { + var72 = 0.10161088917815142; + } else { + if (input[10] > 77.50000000000001) { + var72 = -0.0792333078055653; + } else { + if (input[7] > 23.500000000000004) { + if (input[0] > 2882.5000000000005) { + var72 = -0.06672020005240323; + } else { + var72 = 0.08831457502630258; + } + } else { + if (input[8] > 2592.5000000000005) { + var72 = -0.052617701047376654; + } else { + if (input[10] > 29.500000000000004) { + var72 = 0.08499327690298047; + } else { + if (input[2] > 12.500000000000002) { + if (input[9] > 41.50000000000001) { + var72 = 0.12880460816709416; + } else { + if (input[9] > 25.500000000000004) { + if (input[4] > 11.500000000000002) { + var72 = -0.064099222705728; + } else { + var72 = 0.044332487521538365; + } + } else { + if (input[0] > 2882.5000000000005) { + var72 = 0.031099546885005065; + } else { + var72 = 0.12938467051623853; + } + } + } + } else { + if (input[0] > 4221.500000000001) { + var72 = -0.0928676413498701; + } else { + if (input[9] > 30.500000000000004) { + var72 = -0.05781824812803708; + } else { + var72 = 0.07561268901778094; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } else { + if (input[8] > 711.5000000000001) { + if (input[2] > 22.500000000000004) { + var72 = -0.06648105454098469; + } else { + var72 = 0.05985487552383097; + } + } else { + var72 = -0.13070190291919334; + } + } + } else { + if (input[116] > 1e-35) { + if (input[10] > 38.50000000000001) { + var72 = 0.05282385499619401; + } else { + if (input[1] > 66.50000000000001) { + var72 = 0.048802929108006314; + } else { + if (input[2] > 4.500000000000001) { + if (input[0] > 4593.500000000001) { + var72 = 0.027885690791379255; + } else { + var72 = -0.08407126408362446; + } + } else { + var72 = 0.014432924125571093; + } + } + } + } else { + var72 = -0.00009903435845205118; + } + } + let var73: number; + if (input[76] > 1e-35) { + var73 = -0.06307875292162934; + } else { + if (input[21] > 1e-35) { + if (input[7] > 10.500000000000002) { + if (input[10] > 4.500000000000001) { + if (input[8] > 944.5000000000001) { + if (input[0] > 3655.5000000000005) { + var73 = 0.013633653464240465; + } else { + var73 = -0.10164319411983509; + } + } else { + var73 = -0.1228424374328996; + } + } else { + if (input[1] > 26.500000000000004) { + if (input[2] > 28.500000000000004) { + var73 = 0.00632864847804078; + } else { + var73 = -0.08393000368134668; + } + } else { + var73 = 0.07870508617440916; + } + } + } else { + if (input[284] > 1e-35) { + var73 = 0.1092302727710421; + } else { + var73 = -0.0025505047582483234; + } + } + } else { + if (input[248] > 1e-35) { + var73 = 0.07101822393621864; + } else { + if (input[274] > 1e-35) { + var73 = -0.06621099406425579; + } else { + if (input[1] > 26.500000000000004) { + if (input[1] > 28.500000000000004) { + var73 = 0.0003077044909372931; + } else { + if (input[10] > 2.5000000000000004) { + if (input[0] > 3770.5000000000005) { + var73 = 0.025081789181021243; + } else { + var73 = -0.014813325803582618; + } + } else { + if (input[9] > 33.50000000000001) { + var73 = -0.033466921233840194; + } else { + if (input[3] > 12.500000000000002) { + if (input[23] > 1e-35) { + var73 = 0.11926990418060353; + } else { + var73 = 0.01852125513565268; + } + } else { + var73 = 0.0975367595927343; + } + } + } + } + } else { + if (input[5] > 3325.5000000000005) { + if (input[8] > 892.5000000000001) { + if (input[133] > 1e-35) { + var73 = -0.1178464984373743; + } else { + if (input[283] > 1e-35) { + var73 = 0.043370859226927405; + } else { + if (input[5] > 4320.500000000001) { + var73 = -0.01103141226366587; + } else { + if (input[8] > 1104.5000000000002) { + var73 = -0.023053423988095886; + } else { + var73 = -0.0734238953804657; + } + } + } + } + } else { + if (input[6] > 18.500000000000004) { + if (input[8] > 85.50000000000001) { + var73 = 0.000579145585864887; + } else { + var73 = 0.03389152834202143; + } + } else { + if (input[128] > 1e-35) { + var73 = -0.14527722052568462; + } else { + if (input[210] > 1e-35) { + var73 = -0.08915971541902741; + } else { + if (input[7] > 9.500000000000002) { + var73 = -0.03307314577076116; + } else { + if (input[18] > 1e-35) { + var73 = -0.05521712302023565; + } else { + var73 = 0.009315605032770029; + } + } + } + } + } + } + } else { + var73 = 0.0036332551852289933; + } + } + } + } + } + } + let var74: number; + if (input[0] > 689.5000000000001) { + if (input[5] > 768.5000000000001) { + if (input[20] > 1e-35) { + if (input[5] > 4368.500000000001) { + var74 = -0.07583539600416284; + } else { + if (input[188] > 1e-35) { + var74 = -0.07042659515500142; + } else { + if (input[23] > 1e-35) { + if (input[0] > 3807.5000000000005) { + var74 = -0.011038193049597113; + } else { + var74 = 0.08154028164397753; + } + } else { + if (input[1] > 85.50000000000001) { + var74 = 0.10259361975201933; + } else { + var74 = 0.011640408330521594; + } + } + } + } + } else { + var74 = -0.00023319159023748508; + } + } else { + if (input[92] > 1e-35) { + var74 = 0.13771692859530546; + } else { + var74 = 0.022860029819654806; + } + } + } else { + if (input[1] > 22.500000000000004) { + if (input[1] > 24.500000000000004) { + if (input[2] > 96.50000000000001) { + var74 = 0.09967230141007705; + } else { + if (input[30] > 1e-35) { + var74 = -0.08888529037551285; + } else { + var74 = -0.008615931385397808; + } + } + } else { + if (input[10] > 5.500000000000001) { + if (input[4] > 36.50000000000001) { + var74 = 0.08284665960761373; + } else { + var74 = -0.029292565021289504; + } + } else { + if (input[7] > 7.500000000000001) { + var74 = -0.09945093355204493; + } else { + var74 = -0.008381393701708593; + } + } + } + } else { + if (input[20] > 1e-35) { + var74 = -0.04218678460370465; + } else { + if (input[10] > 6.500000000000001) { + if (input[9] > 2.5000000000000004) { + if (input[1] > 13.500000000000002) { + if (input[8] > 143.50000000000003) { + if (input[4] > 7.500000000000001) { + if (input[2] > 36.50000000000001) { + var74 = 0.07585582641438211; + } else { + if (input[8] > 284.50000000000006) { + var74 = -0.029387993239886723; + } else { + var74 = 0.07716738177321587; + } + } + } else { + if (input[1] > 18.500000000000004) { + var74 = 0.026745348497993746; + } else { + var74 = 0.1427429617069753; + } + } + } else { + if (input[9] > 16.500000000000004) { + if (input[9] > 33.50000000000001) { + var74 = 0.02337306890530338; + } else { + var74 = -0.10390355904767366; + } + } else { + var74 = 0.07390521199638532; + } + } + } else { + var74 = -0.06788247515155237; + } + } else { + var74 = -0.04201446383470994; + } + } else { + if (input[2] > 25.500000000000004) { + if (input[2] > 29.500000000000004) { + if (input[8] > 227.50000000000003) { + var74 = -0.06360325615644084; + } else { + var74 = 0.04342192339836601; + } + } else { + var74 = -0.10598779152030145; + } + } else { + var74 = 0.05253384605768211; + } + } + } + } + } + let var75: number; + if (input[3] > 7.500000000000001) { + if (input[157] > 1e-35) { + var75 = -0.07514182877923786; + } else { + var75 = 0.000636205502279271; + } + } else { + if (input[129] > 1e-35) { + if (input[0] > 2904.5000000000005) { + if (input[0] > 4004.5000000000005) { + var75 = 0.028692053800951845; + } else { + var75 = 0.14081686716133598; + } + } else { + var75 = -0.03316566526940354; + } + } else { + if (input[186] > 1e-35) { + if (input[0] > 2653.5000000000005) { + var75 = 0.0037139292567243084; + } else { + var75 = 0.12662311031652707; + } + } else { + if (input[107] > 1e-35) { + if (input[0] > 612.5000000000001) { + var75 = 0.01202688580305612; + } else { + var75 = 0.0993509141454483; + } + } else { + if (input[203] > 1e-35) { + if (input[1] > 77.50000000000001) { + var75 = 0.043935495082738626; + } else { + var75 = -0.05639305759669704; + } + } else { + if (input[247] > 1e-35) { + var75 = -0.06770766046891649; + } else { + if (input[105] > 1e-35) { + if (input[19] > 1e-35) { + var75 = 0.10331836202616368; + } else { + var75 = 0.0006926658459781341; + } + } else { + if (input[96] > 1e-35) { + var75 = 0.05361846065599475; + } else { + if (input[127] > 1e-35) { + if (input[0] > 2723.5000000000005) { + if (input[1] > 54.50000000000001) { + var75 = -0.0741403257305367; + } else { + var75 = 0.022900127535540854; + } + } else { + if (input[7] > 3.5000000000000004) { + var75 = 0.038110741403836294; + } else { + var75 = 0.14618649985842758; + } + } + } else { + if (input[5] > 3921.5000000000005) { + if (input[1] > 110.50000000000001) { + var75 = -0.09552842289807008; + } else { + if (input[1] > 27.500000000000004) { + var75 = 0.012505935885798007; + } else { + var75 = -0.020509603428689526; + } + } + } else { + if (input[282] > 1e-35) { + if (input[9] > 45.50000000000001) { + if (input[6] > 5.500000000000001) { + var75 = -0.1046104767723845; + } else { + var75 = 0.031388606992301074; + } + } else { + if (input[8] > 114.50000000000001) { + if (input[9] > 17.500000000000004) { + if (input[9] > 22.500000000000004) { + if (input[1] > 32.50000000000001) { + var75 = 0.023466328488582572; + } else { + var75 = 0.11730925774586994; + } + } else { + var75 = -0.04771965631104874; + } + } else { + var75 = 0.17059689880751394; + } + } else { + var75 = -0.08181850955999449; + } + } + } else { + if (input[26] > 1e-35) { + var75 = -0.12727482696678769; + } else { + var75 = -0.014343123272734182; + } + } + } + } + } + } + } + } + } + } + } + } + let var76: number; + if (input[147] > 1e-35) { + if (input[1] > 53.50000000000001) { + var76 = -0.0993064321015924; + } else { + if (input[0] > 2604.5000000000005) { + if (input[0] > 3629.5000000000005) { + var76 = -0.02763546051134888; + } else { + var76 = 0.06423344777499343; + } + } else { + var76 = -0.064606430904295; + } + } + } else { + if (input[302] > 1e-35) { + if (input[10] > 2.5000000000000004) { + if (input[10] > 47.50000000000001) { + var76 = 0.049825139823021586; + } else { + if (input[7] > 22.500000000000004) { + var76 = -0.01131680751379858; + } else { + if (input[0] > 2579.5000000000005) { + var76 = -0.10673674485369694; + } else { + var76 = -0.015387212937189957; + } + } + } + } else { + var76 = 0.04347325151148724; + } + } else { + if (input[179] > 1e-35) { + var76 = -0.05788885608624092; + } else { + if (input[84] > 1e-35) { + if (input[9] > 6.500000000000001) { + if (input[2] > 43.50000000000001) { + var76 = 0.0650355590939066; + } else { + var76 = -0.0473332870892226; + } + } else { + var76 = -0.09699315983340703; + } + } else { + if (input[288] > 1e-35) { + if (input[88] > 1e-35) { + var76 = 0.11139543329789044; + } else { + if (input[126] > 1e-35) { + var76 = -0.09726928633696198; + } else { + if (input[8] > 149.50000000000003) { + if (input[9] > 46.50000000000001) { + if (input[4] > 1.5000000000000002) { + if (input[8] > 1861.5000000000002) { + var76 = 0.06370903833231022; + } else { + if (input[10] > 29.500000000000004) { + var76 = 0.03415223859607161; + } else { + if (input[10] > 3.5000000000000004) { + var76 = -0.07415518117873297; + } else { + var76 = -0.0014119203473324082; + } + } + } + } else { + var76 = 0.12617652343819508; + } + } else { + if (input[9] > 41.50000000000001) { + var76 = -0.10311145857176976; + } else { + if (input[8] > 2757.5000000000005) { + var76 = -0.08106484219011428; + } else { + if (input[7] > 71.50000000000001) { + var76 = -0.09783384432091176; + } else { + if (input[1] > 88.50000000000001) { + var76 = 0.06249739709782831; + } else { + if (input[3] > 9.500000000000002) { + if (input[5] > 1601.5000000000002) { + var76 = -0.008884084501608536; + } else { + var76 = 0.061339437777743616; + } + } else { + var76 = -0.042490992675121846; + } + } + } + } + } + } + } else { + if (input[2] > 6.500000000000001) { + if (input[3] > 10.500000000000002) { + var76 = 0.01526664064166223; + } else { + var76 = 0.13534828515415498; + } + } else { + var76 = -0.06985484465894776; + } + } + } + } + } else { + var76 = 0.0005758961943178744; + } + } + } + } + } + let var77: number; + if (input[86] > 1e-35) { + if (input[1] > 23.500000000000004) { + if (input[1] > 29.500000000000004) { + if (input[4] > 16.500000000000004) { + if (input[2] > 31.500000000000004) { + var77 = -0.029152732370514342; + } else { + var77 = 0.07173628916139178; + } + } else { + if (input[1] > 36.50000000000001) { + var77 = -0.08859111297255318; + } else { + var77 = 0.0018030071815630785; + } + } + } else { + var77 = 0.13652461563759322; + } + } else { + var77 = -0.07550137680349367; + } + } else { + if (input[10] > 52.50000000000001) { + if (input[49] > 1e-35) { + var77 = -0.07145140450454163; + } else { + if (input[21] > 1e-35) { + var77 = -0.07422841663493233; + } else { + var77 = 0.006289319702780104; + } + } + } else { + if (input[10] > 40.50000000000001) { + if (input[9] > 59.50000000000001) { + if (input[19] > 1e-35) { + if (input[13] > 1e-35) { + var77 = 0.11864240653986852; + } else { + if (input[3] > 33.50000000000001) { + var77 = -0.08821209591953476; + } else { + var77 = 0.05706392280054726; + } + } + } else { + var77 = -0.03600088051578915; + } + } else { + if (input[18] > 1e-35) { + if (input[1] > 24.500000000000004) { + var77 = 0.01953613016837112; + } else { + var77 = -0.059781039130025006; + } + } else { + if (input[148] > 1e-35) { + var77 = 0.052668447861325476; + } else { + if (input[3] > 30.500000000000004) { + if (input[9] > 49.50000000000001) { + var77 = 0.07207826841738371; + } else { + if (input[202] > 1e-35) { + var77 = 0.08163917539410503; + } else { + var77 = -0.01319846363832958; + } + } + } else { + if (input[9] > 35.50000000000001) { + if (input[5] > 4134.500000000001) { + if (input[10] > 44.50000000000001) { + var77 = -0.06858280496900336; + } else { + var77 = -0.1781828899516648; + } + } else { + var77 = -0.04024620133969553; + } + } else { + if (input[9] > 10.500000000000002) { + if (input[1] > 22.500000000000004) { + if (input[1] > 37.50000000000001) { + var77 = 0.018232649414147116; + } else { + var77 = -0.04419781124222661; + } + } else { + var77 = 0.05145485182416554; + } + } else { + if (input[1] > 23.500000000000004) { + if (input[0] > 655.5000000000001) { + if (input[5] > 4901.500000000001) { + if (input[10] > 45.50000000000001) { + var77 = 0.11452368095776105; + } else { + var77 = -0.036496437259924026; + } + } else { + var77 = -0.040445338739465486; + } + } else { + var77 = 0.0816572651001145; + } + } else { + var77 = -0.08968914517368663; + } + } + } + } + } + } + } + } else { + var77 = 0.0002826343082585516; + } + } + } + let var78: number; + if (input[189] > 1e-35) { + if (input[0] > 5269.500000000001) { + var78 = -0.08839493050459957; + } else { + if (input[10] > 85.50000000000001) { + var78 = 0.10046908365702462; + } else { + if (input[8] > 2592.5000000000005) { + var78 = -0.09632233975926387; + } else { + if (input[8] > 2000.5000000000002) { + var78 = 0.10282992953871627; + } else { + if (input[8] > 1266.5000000000002) { + if (input[9] > 34.50000000000001) { + var78 = 0.035504970430426296; + } else { + if (input[1] > 31.500000000000004) { + var78 = -0.1133764813142531; + } else { + var78 = -0.01138280942244812; + } + } + } else { + if (input[8] > 1125.5000000000002) { + var78 = 0.09800530246229806; + } else { + var78 = 0.016170419267589393; + } + } + } + } + } + } + } else { + if (input[218] > 1e-35) { + if (input[9] > 99.50000000000001) { + if (input[9] > 101.50000000000001) { + if (input[9] > 124.50000000000001) { + var78 = 0.07316772160107896; + } else { + var78 = -0.059095014819051765; + } + } else { + var78 = 0.17859437315769733; + } + } else { + if (input[2] > 1.5000000000000002) { + if (input[9] > 86.50000000000001) { + var78 = -0.09150209066166894; + } else { + if (input[8] > 3084.0000000000005) { + var78 = -0.05443972593168094; + } else { + if (input[1] > 65.50000000000001) { + if (input[10] > 11.500000000000002) { + if (input[9] > 33.50000000000001) { + var78 = -0.04449234460408263; + } else { + var78 = 0.05568837973347338; + } + } else { + var78 = -0.12362324875024472; + } + } else { + if (input[1] > 41.50000000000001) { + if (input[10] > 12.500000000000002) { + if (input[8] > 1336.5000000000002) { + var78 = 0.12741077850267066; + } else { + var78 = 0.007372371864985329; + } + } else { + if (input[2] > 39.50000000000001) { + var78 = 0.02295917234617787; + } else { + var78 = 0.14966532083907075; + } + } + } else { + if (input[1] > 39.50000000000001) { + var78 = -0.06685557815340279; + } else { + if (input[10] > 22.500000000000004) { + if (input[2] > 52.50000000000001) { + var78 = -0.02511861881285652; + } else { + if (input[1] > 27.500000000000004) { + var78 = 0.08683660011672288; + } else { + var78 = 0.02956214835267301; + } + } + } else { + if (input[9] > 15.500000000000002) { + var78 = -0.016538805462996232; + } else { + var78 = 0.04352738094981517; + } + } + } + } + } + } + } + } else { + var78 = -0.05561856645643868; + } + } + } else { + if (input[9] > 170.50000000000003) { + var78 = -0.07996752635874248; + } else { + if (input[179] > 1e-35) { + var78 = -0.09065975936933919; + } else { + var78 = -0.00042817975060427177; + } + } + } + } + let var79: number; + if (input[39] > 1e-35) { + if (input[4] > 25.500000000000004) { + var79 = 0.03443173196222934; + } else { + var79 = -0.06554248341270724; + } + } else { + if (input[32] > 1e-35) { + if (input[8] > 90.50000000000001) { + if (input[4] > 67.50000000000001) { + if (input[4] > 86.50000000000001) { + var79 = -0.0013415395759330318; + } else { + var79 = 0.12950978489563347; + } + } else { + if (input[1] > 22.500000000000004) { + if (input[10] > 19.500000000000004) { + if (input[4] > 30.500000000000004) { + if (input[9] > 41.50000000000001) { + var79 = 0.002297618040307216; + } else { + var79 = -0.12522800128774994; + } + } else { + if (input[4] > 8.500000000000002) { + if (input[8] > 1075.5000000000002) { + var79 = -0.015297257305397608; + } else { + var79 = 0.09651828834062742; + } + } else { + var79 = -0.06636003334371929; + } + } + } else { + if (input[10] > 11.500000000000002) { + var79 = 0.17631616138309397; + } else { + if (input[0] > 1639.5000000000002) { + var79 = 0.00003804386478092585; + } else { + var79 = -0.09099296398683193; + } + } + } + } else { + var79 = -0.06874415876172972; + } + } + } else { + if (input[0] > 2151.5000000000005) { + var79 = -0.1311264883406766; + } else { + var79 = 0.00809052010141122; + } + } + } else { + if (input[253] > 1e-35) { + var79 = -0.06338558211939296; + } else { + if (input[178] > 1e-35) { + if (input[2] > 25.500000000000004) { + if (input[2] > 30.500000000000004) { + if (input[0] > 2151.5000000000005) { + if (input[10] > 10.500000000000002) { + if (input[0] > 3615.5000000000005) { + var79 = 0.045038497754638605; + } else { + var79 = -0.07770167665661752; + } + } else { + var79 = -0.08596294280650517; + } + } else { + var79 = 0.08538655727027213; + } + } else { + var79 = 0.09829076418590559; + } + } else { + if (input[1] > 39.50000000000001) { + if (input[9] > 1.5000000000000002) { + var79 = 0.054627956617973275; + } else { + if (input[1] > 61.50000000000001) { + var79 = -0.11994465088415499; + } else { + if (input[4] > 8.500000000000002) { + var79 = 0.06676200239406452; + } else { + var79 = -0.027503148069376867; + } + } + } + } else { + if (input[8] > 676.5000000000001) { + var79 = -0.10363964928357075; + } else { + if (input[4] > 8.500000000000002) { + var79 = -0.07589816227175682; + } else { + var79 = 0.034664436544646814; + } + } + } + } + } else { + if (input[1] > 159.50000000000003) { + if (input[6] > 25.500000000000004) { + var79 = 0.009093153189012338; + } else { + var79 = -0.06119765876605404; + } + } else { + var79 = 0.0004668642103528348; + } + } + } + } + } + let var80: number; + if (input[223] > 1e-35) { + if (input[1] > 31.500000000000004) { + if (input[8] > 711.5000000000001) { + var80 = -0.10100794502567233; + } else { + var80 = 0.08000205636470442; + } + } else { + var80 = -0.11945419826856896; + } + } else { + if (input[113] > 1e-35) { + var80 = -0.06105445938688056; + } else { + if (input[167] > 1e-35) { + if (input[0] > 3928.5000000000005) { + var80 = 0.1224302423880318; + } else { + var80 = -0.01875566982911468; + } + } else { + if (input[222] > 1e-35) { + if (input[1] > 8.500000000000002) { + if (input[1] > 24.500000000000004) { + if (input[4] > 3.5000000000000004) { + if (input[0] > 725.5000000000001) { + if (input[0] > 1682.5000000000002) { + if (input[0] > 2860.5000000000005) { + var80 = 0.0019277012166729114; + } else { + if (input[1] > 28.500000000000004) { + var80 = -0.054445821715687494; + } else { + var80 = 0.045645722976713245; + } + } + } else { + if (input[30] > 1e-35) { + var80 = 0.13402660155331655; + } else { + var80 = 0.008921176001777645; + } + } + } else { + var80 = -0.058547426505451076; + } + } else { + var80 = 0.08841202222426625; + } + } else { + if (input[1] > 22.500000000000004) { + if (input[10] > 9.500000000000002) { + var80 = -0.13526418192218206; + } else { + var80 = -0.03266013432583145; + } + } else { + if (input[1] > 20.500000000000004) { + if (input[4] > 27.500000000000004) { + var80 = 0.0007263224246135398; + } else { + var80 = 0.12450043268647056; + } + } else { + if (input[1] > 17.500000000000004) { + if (input[9] > 1.5000000000000002) { + var80 = -0.11575657261278308; + } else { + var80 = -0.01530376565862095; + } + } else { + if (input[4] > 13.500000000000002) { + if (input[4] > 22.500000000000004) { + var80 = -0.01995960178292952; + } else { + var80 = 0.11216586049153021; + } + } else { + var80 = -0.10050961087149474; + } + } + } + } + } + } else { + var80 = 0.08848063368485726; + } + } else { + if (input[30] > 1e-35) { + if (input[224] > 1e-35) { + if (input[1] > 52.50000000000001) { + var80 = 0.10303451081526649; + } else { + var80 = -0.01375730267020699; + } + } else { + if (input[1] > 28.500000000000004) { + if (input[2] > 20.500000000000004) { + var80 = -0.043799548968209395; + } else { + var80 = -0.12451444314954115; + } + } else { + if (input[4] > 12.500000000000002) { + var80 = -0.03838117361958468; + } else { + var80 = 0.06504990789767144; + } + } + } + } else { + if (input[57] > 1e-35) { + var80 = 0.06890006938293915; + } else { + var80 = 0.0003914274695562949; + } + } + } + } + } + } + let var81: number; + if (input[53] > 1e-35) { + if (input[4] > 11.500000000000002) { + if (input[8] > 617.5000000000001) { + if (input[2] > 41.50000000000001) { + var81 = 0.004271749009686975; + } else { + var81 = -0.10523878297127605; + } + } else { + var81 = 0.04633982158107851; + } + } else { + var81 = -0.10349713975483057; + } + } else { + if (input[183] > 1e-35) { + if (input[15] > 1e-35) { + var81 = -0.08655730561951676; + } else { + if (input[8] > 919.5000000000001) { + var81 = -0.0676453705610183; + } else { + if (input[7] > 18.500000000000004) { + var81 = -0.027787974193650575; + } else { + var81 = 0.08012784576991301; + } + } + } + } else { + if (input[227] > 1e-35) { + if (input[1] > 6.500000000000001) { + if (input[3] > 8.500000000000002) { + if (input[210] > 1e-35) { + var81 = 0.07185850683316512; + } else { + if (input[8] > 201.50000000000003) { + if (input[8] > 348.50000000000006) { + if (input[23] > 1e-35) { + if (input[8] > 1049.5000000000002) { + var81 = -0.03473877164537313; + } else { + if (input[8] > 719.5000000000001) { + var81 = 0.10471053866934404; + } else { + var81 = 0.008236107678382981; + } + } + } else { + if (input[4] > 57.50000000000001) { + var81 = 0.09412219478825269; + } else { + if (input[10] > 66.50000000000001) { + var81 = -0.13884338641811986; + } else { + if (input[10] > 19.500000000000004) { + if (input[10] > 22.500000000000004) { + if (input[0] > 2490.5000000000005) { + var81 = -0.040681323751002293; + } else { + var81 = 0.06374650297561021; + } + } else { + var81 = 0.12884615227401788; + } + } else { + if (input[10] > 5.500000000000001) { + var81 = -0.0887517295786972; + } else { + if (input[8] > 597.5000000000001) { + if (input[18] > 1e-35) { + var81 = -0.05474068967150784; + } else { + var81 = 0.03744700650806603; + } + } else { + var81 = -0.07846396348680855; + } + } + } + } + } + } + } else { + if (input[1] > 42.50000000000001) { + var81 = 0.018972315810821302; + } else { + var81 = 0.10953621007604744; + } + } + } else { + if (input[5] > 4439.500000000001) { + var81 = 0.010999776705494586; + } else { + if (input[1] > 40.50000000000001) { + var81 = -0.12394200059775967; + } else { + if (input[10] > 2.5000000000000004) { + var81 = 0.013528093962849453; + } else { + var81 = -0.09222088417048682; + } + } + } + } + } + } else { + var81 = -0.12662967149701485; + } + } else { + var81 = 0.09327296405849603; + } + } else { + if (input[3] > 99.50000000000001) { + var81 = -0.013581954439986752; + } else { + var81 = 0.0005526498251862075; + } + } + } + } + let var82: number; + if (input[187] > 1e-35) { + if (input[243] > 1e-35) { + var82 = -0.08392792551692502; + } else { + if (input[10] > 68.50000000000001) { + var82 = 0.07871769409454053; + } else { + if (input[10] > 8.500000000000002) { + if (input[10] > 16.500000000000004) { + if (input[2] > 17.500000000000004) { + if (input[3] > 31.500000000000004) { + if (input[91] > 1e-35) { + if (input[10] > 21.500000000000004) { + if (input[10] > 33.50000000000001) { + if (input[10] > 48.50000000000001) { + var82 = -0.0825306209711224; + } else { + var82 = 0.049559996084532945; + } + } else { + var82 = -0.1064938580886302; + } + } else { + var82 = 0.03353240732240275; + } + } else { + var82 = 0.045985370399163464; + } + } else { + if (input[1] > 42.50000000000001) { + if (input[4] > 20.500000000000004) { + var82 = 0.16966001471529374; + } else { + if (input[1] > 57.50000000000001) { + var82 = -0.005772777673676247; + } else { + var82 = 0.09383677041525058; + } + } + } else { + if (input[8] > 747.5000000000001) { + var82 = 0.054068175469351235; + } else { + var82 = -0.049968216310277036; + } + } + } + } else { + if (input[8] > 753.5000000000001) { + var82 = -0.0679383555784074; + } else { + if (input[4] > 8.500000000000002) { + var82 = -0.059757341189735386; + } else { + var82 = 0.05701083682780414; + } + } + } + } else { + var82 = -0.052497281448921164; + } + } else { + if (input[6] > 12.500000000000002) { + if (input[8] > 969.5000000000001) { + if (input[4] > 23.500000000000004) { + var82 = 0.05820296128730006; + } else { + var82 = -0.1063042385102475; + } + } else { + if (input[1] > 49.50000000000001) { + if (input[8] > 302.50000000000006) { + var82 = 0.15340611616954566; + } else { + var82 = 0.04385036188666874; + } + } else { + if (input[0] > 4449.500000000001) { + var82 = -0.02110897605541555; + } else { + if (input[1] > 24.500000000000004) { + if (input[2] > 17.500000000000004) { + var82 = 0.004840354641006495; + } else { + var82 = 0.09967827580276283; + } + } else { + var82 = 0.11605363537391578; + } + } + } + } + } else { + if (input[9] > 19.500000000000004) { + var82 = -0.0735831692725717; + } else { + var82 = 0.019973331823355176; + } + } + } + } + } + } else { + if (input[306] > 1e-35) { + if (input[149] > 1e-35) { + var82 = -0.08968948874343531; + } else { + if (input[8] > 1094.5000000000002) { + if (input[10] > 15.500000000000002) { + var82 = -0.02442182361342386; + } else { + var82 = 0.10334853004243093; + } + } else { + var82 = -0.030431948680167104; + } + } + } else { + var82 = -0.0000956078595250818; + } + } + let var83: number; + if (input[294] > 1e-35) { + if (input[1] > 26.500000000000004) { + if (input[0] > 4078.5000000000005) { + var83 = -0.040232505718244854; + } else { + if (input[0] > 3030.5000000000005) { + var83 = 0.0634109586813073; + } else { + var83 = -0.04043617034245621; + } + } + } else { + var83 = -0.06385323610738443; + } + } else { + if (input[120] > 1e-35) { + if (input[4] > 18.500000000000004) { + var83 = -0.007859096946435131; + } else { + var83 = 0.07282728486115758; + } + } else { + if (input[229] > 1e-35) { + if (input[0] > 2952.5000000000005) { + if (input[17] > 1e-35) { + var83 = 0.05515771679628051; + } else { + var83 = -0.04214471312668263; + } + } else { + var83 = -0.09589322222261765; + } + } else { + if (input[193] > 1e-35) { + var83 = -0.05056345906812831; + } else { + if (input[121] > 1e-35) { + if (input[243] > 1e-35) { + var83 = 0.14857706653119385; + } else { + if (input[4] > 9.500000000000002) { + if (input[1] > 26.500000000000004) { + if (input[2] > 59.50000000000001) { + var83 = -0.08152604001147906; + } else { + if (input[11] > 1e-35) { + var83 = 0.09132936522356462; + } else { + if (input[15] > 1e-35) { + if (input[4] > 23.500000000000004) { + var83 = 0.13100930780107503; + } else { + if (input[10] > 25.500000000000004) { + var83 = 0.05921074710011526; + } else { + var83 = -0.07226005736695183; + } + } + } else { + if (input[0] > 3304.5000000000005) { + if (input[0] > 3707.5000000000005) { + if (input[0] > 4053.5000000000005) { + var83 = 0.0009447118243153454; + } else { + var83 = -0.09820565036865991; + } + } else { + var83 = 0.057146909749745546; + } + } else { + if (input[0] > 2115.5000000000005) { + var83 = -0.12331216726611678; + } else { + var83 = 0.007281983677694285; + } + } + } + } + } + } else { + if (input[2] > 56.50000000000001) { + var83 = 0.012310154675612615; + } else { + var83 = -0.08873665774670461; + } + } + } else { + if (input[6] > 25.500000000000004) { + var83 = 0.134708740821879; + } else { + if (input[9] > 5.500000000000001) { + var83 = -0.0805901581148979; + } else { + if (input[224] > 1e-35) { + var83 = -0.063684477784257; + } else { + if (input[7] > 2.5000000000000004) { + if (input[19] > 1e-35) { + var83 = 0.10842593386554122; + } else { + if (input[2] > 13.500000000000002) { + var83 = 0.06466798320378395; + } else { + var83 = -0.08578130788886655; + } + } + } else { + var83 = -0.03590892078300114; + } + } + } + } + } + } + } else { + var83 = 0.0003499894043880708; + } + } + } + } + } + let var84: number; + if (input[134] > 1e-35) { + if (input[6] > 50.50000000000001) { + if (input[0] > 3601.5000000000005) { + var84 = 0.10839808814624702; + } else { + var84 = -0.028043875308180352; + } + } else { + if (input[7] > 30.500000000000004) { + if (input[8] > 932.5000000000001) { + var84 = -0.007478368069393829; + } else { + var84 = -0.09066751344326617; + } + } else { + if (input[0] > 3588.5000000000005) { + if (input[5] > 4748.500000000001) { + var84 = 0.04035247751736232; + } else { + if (input[0] > 4255.500000000001) { + var84 = -0.1310865624507367; + } else { + if (input[0] > 4004.5000000000005) { + var84 = 0.06647367311982634; + } else { + var84 = -0.08339693352955757; + } + } + } + } else { + if (input[4] > 10.500000000000002) { + if (input[1] > 34.50000000000001) { + var84 = -0.011618902907510411; + } else { + var84 = 0.1114646660406691; + } + } else { + if (input[10] > 2.5000000000000004) { + if (input[0] > 3072.5000000000005) { + var84 = 0.09356028223727986; + } else { + var84 = -0.03811765057032162; + } + } else { + var84 = -0.09456215497345526; + } + } + } + } + } + } else { + if (input[280] > 1e-35) { + if (input[7] > 70.50000000000001) { + var84 = 0.10322956436499003; + } else { + if (input[2] > 22.500000000000004) { + if (input[1] > 83.50000000000001) { + var84 = 0.1146142460964847; + } else { + if (input[1] > 62.50000000000001) { + var84 = -0.09679869865322362; + } else { + if (input[9] > 71.50000000000001) { + var84 = -0.07377580769927583; + } else { + if (input[4] > 19.500000000000004) { + if (input[0] > 4571.500000000001) { + var84 = -0.039046426387852974; + } else { + var84 = 0.04558778688367152; + } + } else { + var84 = 0.11220830937352602; + } + } + } + } + } else { + if (input[7] > 5.500000000000001) { + if (input[9] > 17.500000000000004) { + if (input[8] > 1067.5000000000002) { + var84 = 0.03261697816211156; + } else { + if (input[15] > 1e-35) { + var84 = 0.02586252542264368; + } else { + if (input[2] > 14.500000000000002) { + var84 = -0.016420452667484604; + } else { + var84 = -0.1011799626006976; + } + } + } + } else { + var84 = -0.13787471318963773; + } + } else { + if (input[6] > 4.500000000000001) { + if (input[8] > 427.50000000000006) { + if (input[10] > 36.50000000000001) { + var84 = 0.010193588102560583; + } else { + var84 = 0.11748729525930773; + } + } else { + var84 = -0.04468162226743652; + } + } else { + var84 = -0.028365274393617957; + } + } + } + } + } else { + if (input[71] > 1e-35) { + var84 = 0.05115139346588793; + } else { + var84 = -0.0001510425316936658; + } + } + } + let var85: number; + if (input[298] > 1e-35) { + if (input[8] > 81.50000000000001) { + if (input[8] > 119.50000000000001) { + if (input[4] > 64.50000000000001) { + var85 = 0.09072192054181037; + } else { + if (input[9] > 72.50000000000001) { + if (input[8] > 1094.5000000000002) { + var85 = 0.020637047900190317; + } else { + var85 = -0.1017300802134141; + } + } else { + if (input[1] > 23.500000000000004) { + if (input[9] > 12.500000000000002) { + if (input[0] > 2815.5000000000005) { + if (input[0] > 3183.5000000000005) { + if (input[3] > 23.500000000000004) { + if (input[3] > 45.50000000000001) { + if (input[4] > 48.50000000000001) { + var85 = -0.04632587527094407; + } else { + var85 = 0.08603684785510396; + } + } else { + var85 = -0.05101401015448496; + } + } else { + var85 = 0.025466432054358498; + } + } else { + var85 = -0.07897811963329214; + } + } else { + if (input[6] > 13.500000000000002) { + if (input[10] > 26.500000000000004) { + var85 = 0.020385355430046367; + } else { + var85 = 0.12032592051335252; + } + } else { + var85 = -0.012387370292173013; + } + } + } else { + if (input[2] > 23.500000000000004) { + var85 = -0.12568545484492677; + } else { + var85 = -0.022261190943521976; + } + } + } else { + if (input[8] > 634.5000000000001) { + if (input[8] > 857.5000000000001) { + var85 = 0.043528764484784536; + } else { + var85 = 0.14352071657196003; + } + } else { + var85 = -0.009332833816977268; + } + } + } + } + } else { + var85 = 0.11186782227735846; + } + } else { + var85 = -0.0737365712425554; + } + } else { + if (input[136] > 1e-35) { + if (input[0] > 1937.5000000000002) { + var85 = -0.05649104643152564; + } else { + var85 = 0.03884200719305747; + } + } else { + if (input[42] > 1e-35) { + var85 = -0.07191700385792335; + } else { + if (input[116] > 1e-35) { + if (input[9] > 2.5000000000000004) { + if (input[9] > 17.500000000000004) { + var85 = -0.04103416502526736; + } else { + var85 = 0.04881823954656287; + } + } else { + if (input[4] > 15.500000000000002) { + var85 = 0.009342724662897898; + } else { + if (input[0] > 3969.5000000000005) { + var85 = -0.025637309961309498; + } else { + var85 = -0.12574492012987865; + } + } + } + } else { + if (input[212] > 1e-35) { + if (input[19] > 1e-35) { + var85 = -0.08185697075265091; + } else { + if (input[0] > 2215.5000000000005) { + var85 = 0.030063975892297354; + } else { + if (input[0] > 807.5000000000001) { + var85 = -0.03924325550733229; + } else { + var85 = 0.0415330999189793; + } + } + } + } else { + var85 = -0.00024374664461674863; + } + } + } + } + } + let var86: number; + if (input[3] > 7.500000000000001) { + var86 = 0.0005117490419655908; + } else { + if (input[129] > 1e-35) { + if (input[0] > 2904.5000000000005) { + if (input[0] > 4004.5000000000005) { + var86 = 0.025798416259686565; + } else { + var86 = 0.13251610353146012; + } + } else { + var86 = -0.029900559552677654; + } + } else { + if (input[1] > 81.50000000000001) { + if (input[1] > 110.50000000000001) { + if (input[0] > 4242.500000000001) { + var86 = -0.11098564237775424; + } else { + var86 = 0.000025960925309712775; + } + } else { + if (input[0] > 4177.500000000001) { + if (input[9] > 35.50000000000001) { + var86 = 0.15347826616466054; + } else { + if (input[3] > 4.500000000000001) { + var86 = 0.10379320730958941; + } else { + var86 = -0.008896303020010654; + } + } + } else { + if (input[0] > 3415.5000000000005) { + if (input[0] > 3830.5000000000005) { + var86 = 0.03159791088468647; + } else { + var86 = -0.10612873364104258; + } + } else { + var86 = 0.05059856107348746; + } + } + } + } else { + if (input[133] > 1e-35) { + if (input[2] > 5.500000000000001) { + var86 = -0.02335760775001469; + } else { + var86 = -0.1379386577903324; + } + } else { + if (input[1] > 62.50000000000001) { + if (input[3] > 2.5000000000000004) { + var86 = -0.011164334474672973; + } else { + var86 = -0.06594044410501655; + } + } else { + if (input[207] > 1e-35) { + var86 = -0.1014214372326535; + } else { + if (input[8] > 3.5000000000000004) { + if (input[107] > 1e-35) { + if (input[2] > 6.500000000000001) { + var86 = -0.01725821503981916; + } else { + var86 = 0.05594086838700241; + } + } else { + if (input[203] > 1e-35) { + if (input[1] > 44.50000000000001) { + if (input[1] > 51.50000000000001) { + var86 = -0.04226531631656534; + } else { + var86 = -0.14409800530171432; + } + } else { + var86 = -0.03245576341206398; + } + } else { + if (input[8] > 4214.500000000001) { + var86 = 0.0895409165534886; + } else { + if (input[247] > 1e-35) { + var86 = -0.06506383629143335; + } else { + if (input[118] > 1e-35) { + var86 = -0.07214270121257443; + } else { + if (input[8] > 546.5000000000001) { + var86 = -0.004385020865473831; + } else { + var86 = 0.009321812545248529; + } + } + } + } + } + } + } else { + if (input[0] > 1639.5000000000002) { + if (input[13] > 1e-35) { + var86 = 0.046278501133958524; + } else { + var86 = -0.030835570926968044; + } + } else { + if (input[0] > 493.50000000000006) { + var86 = -0.12794504651610425; + } else { + var86 = 0.009415039807550776; + } + } + } + } + } + } + } + } + } + let var87: number; + if (input[304] > 1e-35) { + var87 = -0.04717777269217453; + } else { + if (input[76] > 1e-35) { + var87 = -0.05813439142128324; + } else { + if (input[1] > 59.50000000000001) { + if (input[0] > 350.50000000000006) { + if (input[53] > 1e-35) { + var87 = -0.09648224457374217; + } else { + if (input[132] > 1e-35) { + var87 = 0.07089308107910267; + } else { + if (input[0] > 2248.5000000000005) { + if (input[5] > 2525.5000000000005) { + if (input[9] > 1.5000000000000002) { + if (input[114] > 1e-35) { + var87 = -0.08595213071749083; + } else { + if (input[9] > 14.500000000000002) { + if (input[9] > 33.50000000000001) { + if (input[285] > 1e-35) { + var87 = 0.10838431695638147; + } else { + if (input[230] > 1e-35) { + var87 = 0.06458713915750626; + } else { + if (input[0] > 3219.5000000000005) { + if (input[3] > 23.500000000000004) { + if (input[9] > 69.50000000000001) { + var87 = 0.050071316251979; + } else { + var87 = -0.006356941111525215; + } + } else { + if (input[6] > 8.500000000000002) { + var87 = -0.0384814076434817; + } else { + if (input[1] > 73.50000000000001) { + if (input[0] > 3746.5000000000005) { + var87 = 0.10217402850540398; + } else { + var87 = -0.048840949025349197; + } + } else { + var87 = -0.03668313197909846; + } + } + } + } else { + if (input[7] > 39.50000000000001) { + var87 = -0.0562642841496003; + } else { + if (input[10] > 2.5000000000000004) { + var87 = 0.09749777369987417; + } else { + var87 = -0.04848223121417616; + } + } + } + } + } + } else { + if (input[0] > 5453.500000000001) { + var87 = 0.08316648226133942; + } else { + var87 = -0.0261979698267618; + } + } + } else { + if (input[212] > 1e-35) { + var87 = 0.09565573198318654; + } else { + if (input[5] > 4814.500000000001) { + if (input[8] > 963.5000000000001) { + if (input[8] > 1514.5000000000002) { + var87 = 0.04837009746506856; + } else { + var87 = -0.09184360565631328; + } + } else { + var87 = 0.0032411047845613606; + } + } else { + if (input[0] > 4733.500000000001) { + var87 = 0.0977378556864798; + } else { + var87 = 0.010776545559325588; + } + } + } + } + } + } else { + var87 = -0.012483310473120218; + } + } else { + var87 = -0.049284121449103935; + } + } else { + var87 = 0.011962641341789565; + } + } + } + } else { + if (input[1] > 67.50000000000001) { + if (input[1] > 77.50000000000001) { + var87 = -0.08380361910948711; + } else { + var87 = 0.07375088778585813; + } + } else { + var87 = -0.1084864186071348; + } + } + } else { + var87 = 0.0007819503469605476; + } + } + } + let var88: number; + if (input[7] > 17.500000000000004) { + if (input[115] > 1e-35) { + var88 = 0.08741852531696623; + } else { + if (input[167] > 1e-35) { + var88 = 0.10078975495600809; + } else { + var88 = -0.0018324767784017562; + } + } + } else { + if (input[290] > 1e-35) { + var88 = -0.0850089851255888; + } else { + if (input[74] > 1e-35) { + if (input[10] > 16.500000000000004) { + var88 = 0.1379733311640402; + } else { + var88 = -0.0038500648529631075; + } + } else { + if (input[6] > 29.500000000000004) { + if (input[8] > 876.5000000000001) { + if (input[0] > 3129.5000000000005) { + if (input[9] > 5.500000000000001) { + if (input[8] > 1765.5000000000002) { + var88 = -0.09360083033774169; + } else { + var88 = 0.061471353193188374; + } + } else { + if (input[10] > 11.500000000000002) { + if (input[10] > 31.500000000000004) { + var88 = -0.015599362579530679; + } else { + if (input[0] > 4593.500000000001) { + var88 = -0.12029549262691491; + } else { + var88 = -0.018917032256501397; + } + } + } else { + var88 = 0.04632831686576592; + } + } + } else { + var88 = 0.06892347785444271; + } + } else { + if (input[4] > 8.500000000000002) { + if (input[10] > 33.50000000000001) { + var88 = -0.05894883236412263; + } else { + var88 = 0.05213944998315824; + } + } else { + var88 = 0.12621779223564986; + } + } + } else { + if (input[243] > 1e-35) { + if (input[6] > 16.500000000000004) { + if (input[0] > 4141.500000000001) { + if (input[0] > 5850.500000000001) { + var88 = 0.07577412405680808; + } else { + var88 = -0.053144737214742235; + } + } else { + if (input[1] > 29.500000000000004) { + if (input[9] > 16.500000000000004) { + var88 = -0.0277076900736147; + } else { + if (input[1] > 65.50000000000001) { + var88 = -0.023587471585763506; + } else { + var88 = 0.10184896592433082; + } + } + } else { + var88 = -0.057699270527916825; + } + } + } else { + var88 = -0.041191811945739454; + } + } else { + if (input[114] > 1e-35) { + if (input[2] > 23.500000000000004) { + var88 = 0.06566902102799584; + } else { + if (input[10] > 25.500000000000004) { + var88 = -0.07033633753181047; + } else { + var88 = -0.01599120398351932; + } + } + } else { + if (input[242] > 1e-35) { + if (input[0] > 2402.5000000000005) { + var88 = -0.08108035861059537; + } else { + var88 = 0.04184690010531078; + } + } else { + if (input[35] > 1e-35) { + if (input[0] > 2904.5000000000005) { + var88 = -0.12431182772561139; + } else { + var88 = 0.01886235886984271; + } + } else { + var88 = 0.0025579594894418116; + } + } + } + } + } + } + } + } + let var89: number; + if (input[8] > 2915.5000000000005) { + if (input[101] > 1e-35) { + var89 = 0.08648323956719083; + } else { + if (input[0] > 93.50000000000001) { + if (input[196] > 1e-35) { + var89 = -0.09509320772734361; + } else { + if (input[4] > 1.5000000000000002) { + if (input[5] > 1106.5000000000002) { + if (input[5] > 1191.5000000000002) { + if (input[283] > 1e-35) { + var89 = -0.11268313808648661; + } else { + if (input[10] > 12.500000000000002) { + if (input[131] > 1e-35) { + var89 = 0.0687641681341721; + } else { + if (input[10] > 102.50000000000001) { + var89 = -0.09667920080214842; + } else { + if (input[4] > 15.500000000000002) { + if (input[8] > 2992.5000000000005) { + if (input[1] > 24.500000000000004) { + if (input[1] > 71.50000000000001) { + var89 = -0.06762578396473291; + } else { + if (input[10] > 65.50000000000001) { + var89 = -0.05226727783610509; + } else { + if (input[282] > 1e-35) { + var89 = 0.09911438410640917; + } else { + if (input[19] > 1e-35) { + var89 = 0.06915156336429933; + } else { + var89 = -0.006565637886508241; + } + } + } + } + } else { + var89 = -0.08344300251849307; + } + } else { + var89 = -0.0928863907927501; + } + } else { + if (input[1] > 60.50000000000001) { + if (input[2] > 17.500000000000004) { + var89 = 0.19428463865406298; + } else { + var89 = 0.016073883020956765; + } + } else { + if (input[13] > 1e-35) { + var89 = 0.06864077097923665; + } else { + var89 = -0.01388867527034731; + } + } + } + } + } + } else { + if (input[0] > 1847.5000000000002) { + var89 = 0.004655280608161356; + } else { + if (input[1] > 40.50000000000001) { + var89 = 0.031406054057765996; + } else { + var89 = 0.12798062439212832; + } + } + } + } + } else { + var89 = 0.09859670536264255; + } + } else { + if (input[10] > 2.5000000000000004) { + if (input[9] > 68.50000000000001) { + var89 = 0.08821759640665892; + } else { + if (input[9] > 32.50000000000001) { + if (input[8] > 3960.0000000000005) { + if (input[1] > 31.500000000000004) { + var89 = -0.0706095614785733; + } else { + var89 = 0.04227164041372561; + } + } else { + var89 = -0.1056906923176064; + } + } else { + if (input[2] > 8.500000000000002) { + if (input[19] > 1e-35) { + var89 = -0.07139533369873902; + } else { + var89 = 0.008952586782921625; + } + } else { + var89 = 0.06086212582180936; + } + } + } + } else { + var89 = -0.0816938490403437; + } + } + } else { + var89 = -0.051224901945956025; + } + } + } else { + var89 = -0.10525399124186095; + } + } + } else { + var89 = 0.000270924147208224; + } + let var90: number; + if (input[122] > 1e-35) { + if (input[0] > 2461.5000000000005) { + if (input[2] > 36.50000000000001) { + var90 = 0.029186512383291244; + } else { + if (input[7] > 1.5000000000000002) { + var90 = -0.14984127276725573; + } else { + if (input[1] > 40.50000000000001) { + var90 = 0.032757060730648144; + } else { + var90 = -0.07675575422749602; + } + } + } + } else { + if (input[6] > 8.500000000000002) { + var90 = 0.10599766037117893; + } else { + var90 = -0.0541423394552156; + } + } + } else { + if (input[1] > 24.500000000000004) { + if (input[103] > 1e-35) { + if (input[8] > 61.50000000000001) { + if (input[17] > 1e-35) { + var90 = -0.051394622947855385; + } else { + var90 = 0.03237141302699347; + } + } else { + var90 = 0.12526173027943244; + } + } else { + var90 = 0.000579473126472788; + } + } else { + if (input[18] > 1e-35) { + if (input[3] > 4.500000000000001) { + if (input[3] > 6.500000000000001) { + if (input[0] > 5453.500000000001) { + var90 = -0.07383912482657777; + } else { + if (input[0] > 5147.500000000001) { + var90 = 0.07008813937042091; + } else { + if (input[10] > 38.50000000000001) { + var90 = -0.06779203808365307; + } else { + var90 = -0.013782769999524498; + } + } + } + } else { + var90 = 0.0880038869117715; + } + } else { + var90 = -0.12846294176070952; + } + } else { + if (input[281] > 1e-35) { + var90 = -0.06810806903850834; + } else { + if (input[10] > 227.50000000000003) { + var90 = -0.08937977001661111; + } else { + if (input[10] > 130.50000000000003) { + var90 = 0.10538920632708033; + } else { + if (input[145] > 1e-35) { + if (input[4] > 6.500000000000001) { + if (input[9] > 16.500000000000004) { + if (input[4] > 18.500000000000004) { + var90 = 0.011036530162093841; + } else { + var90 = -0.11500797478569702; + } + } else { + var90 = 0.03702229366129399; + } + } else { + var90 = 0.07242026683784307; + } + } else { + if (input[189] > 1e-35) { + var90 = 0.03331407112090286; + } else { + if (input[9] > 33.50000000000001) { + if (input[201] > 1e-35) { + var90 = 0.08979610115743614; + } else { + if (input[7] > 57.50000000000001) { + if (input[1] > 20.500000000000004) { + var90 = -0.02608892716555304; + } else { + var90 = 0.09609599320761308; + } + } else { + if (input[9] > 105.50000000000001) { + var90 = -0.06848127135991534; + } else { + var90 = 0.0023675721254089715; + } + } + } + } else { + if (input[86] > 1e-35) { + var90 = -0.11049635625500497; + } else { + var90 = -0.004847764219432233; + } + } + } + } + } + } + } + } + } + } + let var91: number; + if (input[125] > 1e-35) { + if (input[0] > 3969.5000000000005) { + var91 = -0.09462233499115416; + } else { + var91 = 0.05235324508465096; + } + } else { + if (input[17] > 1e-35) { + if (input[49] > 1e-35) { + if (input[10] > 19.500000000000004) { + var91 = -0.030700661288166148; + } else { + var91 = 0.0870883677166864; + } + } else { + if (input[10] > 3.5000000000000004) { + if (input[3] > 18.500000000000004) { + if (input[0] > 3544.5000000000005) { + if (input[188] > 1e-35) { + if (input[9] > 7.500000000000001) { + var91 = 0.03149547314036763; + } else { + var91 = -0.08166208257451366; + } + } else { + if (input[0] > 5850.500000000001) { + var91 = -0.10228136324773157; + } else { + if (input[102] > 1e-35) { + var91 = -0.10572585290676295; + } else { + if (input[8] > 726.5000000000001) { + if (input[5] > 3657.5000000000005) { + var91 = 0.01782894842128785; + } else { + if (input[13] > 1e-35) { + var91 = 0.002680190260979968; + } else { + var91 = 0.1773965720476949; + } + } + } else { + if (input[2] > 72.50000000000001) { + var91 = 0.09090831938627947; + } else { + if (input[1] > 59.50000000000001) { + var91 = -0.12297206702816128; + } else { + if (input[0] > 4977.500000000001) { + var91 = 0.09899015653118268; + } else { + var91 = -0.022207141540838887; + } + } + } + } + } + } + } + } else { + if (input[4] > 32.50000000000001) { + if (input[1] > 34.50000000000001) { + var91 = -0.0675900954187773; + } else { + var91 = 0.012336403425364092; + } + } else { + var91 = -0.0017002325391924573; + } + } + } else { + if (input[6] > 7.500000000000001) { + if (input[1] > 17.500000000000004) { + var91 = -0.02671721777458802; + } else { + var91 = -0.09242452991958029; + } + } else { + if (input[284] > 1e-35) { + var91 = -0.08585691288582491; + } else { + var91 = 0.013332890564324447; + } + } + } + } else { + if (input[4] > 14.500000000000002) { + var91 = -0.005245022074799553; + } else { + if (input[23] > 1e-35) { + var91 = -0.020036720167235768; + } else { + if (input[1] > 29.500000000000004) { + if (input[114] > 1e-35) { + var91 = -0.09289852307936758; + } else { + if (input[116] > 1e-35) { + var91 = -0.09686573010015055; + } else { + if (input[8] > 804.5000000000001) { + var91 = 0.03812547148215318; + } else { + var91 = 0.005162744968176633; + } + } + } + } else { + if (input[9] > 43.50000000000001) { + var91 = -0.059246106396159376; + } else { + var91 = 0.050370113808135275; + } + } + } + } + } + } + } else { + var91 = 0.000794041852811028; + } + } + let var92: number; + if (input[3] > 7.500000000000001) { + var92 = 0.0004981426543104341; + } else { + if (input[9] > 114.50000000000001) { + var92 = 0.05666010099424601; + } else { + if (input[129] > 1e-35) { + if (input[6] > 3.5000000000000004) { + var92 = -0.019061766497948867; + } else { + var92 = 0.07193491146561211; + } + } else { + if (input[186] > 1e-35) { + if (input[0] > 2653.5000000000005) { + var92 = -0.006044199577160493; + } else { + var92 = 0.1147136801028133; + } + } else { + if (input[6] > 85.50000000000001) { + if (input[8] > 847.5000000000001) { + var92 = 0.11486607015912494; + } else { + if (input[9] > 16.500000000000004) { + var92 = -0.08686820858087294; + } else { + var92 = 0.06119632492911875; + } + } + } else { + if (input[127] > 1e-35) { + if (input[0] > 2723.5000000000005) { + if (input[0] > 3682.5000000000005) { + if (input[1] > 38.50000000000001) { + var92 = -0.022230207980026437; + } else { + var92 = 0.1056683690528792; + } + } else { + var92 = -0.05859530800943035; + } + } else { + var92 = 0.06970608927597141; + } + } else { + if (input[7] > 3.5000000000000004) { + if (input[105] > 1e-35) { + var92 = 0.08073568184886762; + } else { + if (input[107] > 1e-35) { + if (input[2] > 6.500000000000001) { + var92 = -0.05177544573528314; + } else { + var92 = 0.05370469772149028; + } + } else { + if (input[1] > 35.50000000000001) { + if (input[0] > 4106.500000000001) { + if (input[9] > 46.50000000000001) { + if (input[0] > 4633.500000000001) { + var92 = 0.15159657923771555; + } else { + var92 = -0.0060542654587671055; + } + } else { + if (input[9] > 5.500000000000001) { + var92 = -0.042808028205051786; + } else { + if (input[1] > 48.50000000000001) { + var92 = -0.010449538258110742; + } else { + var92 = 0.10026907521968294; + } + } + } + } else { + var92 = -0.04249349329714756; + } + } else { + if (input[9] > 42.50000000000001) { + if (input[1] > 19.500000000000004) { + if (input[8] > 852.5000000000001) { + var92 = -0.02272452389409874; + } else { + var92 = -0.11202691218244319; + } + } else { + if (input[5] > 1809.5000000000002) { + var92 = -0.04460413584255906; + } else { + var92 = 0.08196329474205256; + } + } + } else { + if (input[10] > 69.50000000000001) { + var92 = 0.10221481166238167; + } else { + var92 = 0.0004063052701699382; + } + } + } + } + } + } else { + if (input[243] > 1e-35) { + var92 = -0.07563941678849846; + } else { + if (input[18] > 1e-35) { + var92 = 0.02563513231103432; + } else { + var92 = -0.004740081147303786; + } + } + } + } + } + } + } + } + } + let var93: number; + if (input[84] > 1e-35) { + if (input[9] > 6.500000000000001) { + if (input[2] > 43.50000000000001) { + var93 = 0.057446442918106; + } else { + var93 = -0.04404018270156349; + } + } else { + var93 = -0.09282976714550464; + } + } else { + if (input[0] > 384.50000000000006) { + if (input[204] > 1e-35) { + if (input[1] > 62.50000000000001) { + var93 = -0.05930486238817954; + } else { + if (input[1] > 29.500000000000004) { + var93 = 0.06955866121256543; + } else { + if (input[8] > 597.5000000000001) { + var93 = -0.06538593556505168; + } else { + var93 = 0.06212512595497445; + } + } + } + } else { + var93 = 0.00021102929959182257; + } + } else { + if (input[9] > 90.50000000000001) { + var93 = 0.0958061289119631; + } else { + if (input[102] > 1e-35) { + var93 = 0.07172059675638813; + } else { + if (input[1] > 47.50000000000001) { + var93 = -0.03879798603977766; + } else { + if (input[297] > 1e-35) { + var93 = 0.054948234271956144; + } else { + if (input[282] > 1e-35) { + if (input[2] > 6.500000000000001) { + var93 = 0.003805910996312012; + } else { + var93 = 0.09304295674749524; + } + } else { + if (input[11] > 1e-35) { + if (input[18] > 1e-35) { + var93 = 0.11252376801858695; + } else { + if (input[288] > 1e-35) { + var93 = -0.10293901912180432; + } else { + var93 = 0.014669268837893872; + } + } + } else { + if (input[1] > 42.50000000000001) { + var93 = -0.05988274123836837; + } else { + if (input[145] > 1e-35) { + var93 = 0.06142784665288495; + } else { + if (input[3] > 1.5000000000000002) { + if (input[4] > 4.500000000000001) { + if (input[1] > 21.500000000000004) { + if (input[1] > 27.500000000000004) { + if (input[9] > 24.500000000000004) { + var93 = 0.038791154988529926; + } else { + if (input[10] > 22.500000000000004) { + if (input[2] > 19.500000000000004) { + var93 = -0.03366718308159971; + } else { + var93 = 0.11936550608549797; + } + } else { + if (input[1] > 31.500000000000004) { + var93 = -0.07454716789539667; + } else { + var93 = 0.027859650621164217; + } + } + } + } else { + if (input[10] > 10.500000000000002) { + var93 = -0.11806374092321247; + } else { + var93 = -0.03506042229223101; + } + } + } else { + var93 = -0.0007080765837654515; + } + } else { + if (input[10] > 6.500000000000001) { + var93 = -0.028077713664996503; + } else { + if (input[2] > 7.500000000000001) { + var93 = 0.15803724124216814; + } else { + var93 = 0.0351381284833169; + } + } + } + } else { + var93 = -0.07877953381054767; + } + } + } + } + } + } + } + } + } + } + } + let var94: number; + if (input[131] > 1e-35) { + if (input[282] > 1e-35) { + if (input[4] > 23.500000000000004) { + var94 = 0.14144941521975005; + } else { + var94 = 0.0007727806714190652; + } + } else { + if (input[9] > 1.5000000000000002) { + if (input[8] > 2134.5000000000005) { + if (input[2] > 34.50000000000001) { + var94 = 0.10514088112381886; + } else { + if (input[7] > 18.500000000000004) { + var94 = -0.10370643555956745; + } else { + var94 = 0.04093594315421388; + } + } + } else { + if (input[6] > 15.500000000000002) { + if (input[4] > 9.500000000000002) { + if (input[10] > 27.500000000000004) { + if (input[10] > 71.50000000000001) { + var94 = -0.0508129468802936; + } else { + if (input[224] > 1e-35) { + var94 = -0.037816066368733595; + } else { + if (input[10] > 43.50000000000001) { + var94 = 0.07793408602607932; + } else { + var94 = 0.017646166646099453; + } + } + } + } else { + if (input[9] > 3.5000000000000004) { + if (input[9] > 29.500000000000004) { + if (input[17] > 1e-35) { + var94 = 0.036972453794202324; + } else { + var94 = -0.08727431092411866; + } + } else { + if (input[8] > 427.50000000000006) { + if (input[8] > 1278.5000000000002) { + var94 = 0.09475302525132188; + } else { + var94 = -0.03580104945898193; + } + } else { + var94 = 0.08349488283861875; + } + } + } else { + if (input[10] > 3.5000000000000004) { + if (input[0] > 1847.5000000000002) { + if (input[0] > 4280.500000000001) { + if (input[2] > 27.500000000000004) { + var94 = -0.1282448778804823; + } else { + var94 = -0.014395808269207212; + } + } else { + var94 = -0.008940927190750592; + } + } else { + var94 = -0.1459118815453748; + } + } else { + if (input[0] > 4897.500000000001) { + var94 = -0.09733068457286576; + } else { + if (input[1] > 57.50000000000001) { + var94 = 0.06575271409540207; + } else { + var94 = -0.019556422817450115; + } + } + } + } + } + } else { + var94 = -0.10623959222984136; + } + } else { + if (input[18] > 1e-35) { + var94 = 0.11280940901275241; + } else { + if (input[8] > 319.50000000000006) { + if (input[2] > 6.500000000000001) { + var94 = 0.008125645893104896; + } else { + var94 = -0.11084368630465868; + } + } else { + var94 = 0.0584398731508786; + } + } + } + } + } else { + if (input[0] > 350.50000000000006) { + if (input[3] > 83.50000000000001) { + var94 = -0.05854904579626861; + } else { + if (input[4] > 5.500000000000001) { + var94 = 0.02985784951394175; + } else { + var94 = -0.03247600140149334; + } + } + } else { + var94 = -0.11152899295304973; + } + } + } + } else { + var94 = -0.00035424577714215764; + } + let var95: number; + if (input[32] > 1e-35) { + if (input[17] > 1e-35) { + if (input[8] > 359.50000000000006) { + if (input[8] > 804.5000000000001) { + var95 = -0.06563670567578264; + } else { + var95 = 0.067656954313663; + } + } else { + var95 = -0.10388217548685377; + } + } else { + if (input[8] > 2302.5000000000005) { + var95 = 0.07190621943790435; + } else { + if (input[4] > 67.50000000000001) { + var95 = 0.060020507643618604; + } else { + if (input[4] > 38.50000000000001) { + var95 = -0.08707253184321638; + } else { + if (input[2] > 11.500000000000002) { + if (input[2] > 16.500000000000004) { + if (input[1] > 31.500000000000004) { + if (input[1] > 59.50000000000001) { + var95 = -0.06568134366461277; + } else { + if (input[8] > 1075.5000000000002) { + var95 = -0.004768057709758692; + } else { + var95 = 0.11785959165999467; + } + } + } else { + var95 = -0.05080221682879267; + } + } else { + var95 = 0.14814206127494542; + } + } else { + var95 = -0.07241946332311736; + } + } + } + } + } + } else { + if (input[253] > 1e-35) { + var95 = -0.058893562861261274; + } else { + if (input[4] > 61.50000000000001) { + if (input[283] > 1e-35) { + if (input[10] > 23.500000000000004) { + var95 = -0.02471195342450034; + } else { + var95 = 0.11866056464409412; + } + } else { + if (input[10] > 44.50000000000001) { + if (input[1] > 16.500000000000004) { + if (input[8] > 2640.0000000000005) { + var95 = -0.10741850739482771; + } else { + var95 = 0.010051635824944; + } + } else { + var95 = 0.12502069436017124; + } + } else { + if (input[8] > 1971.5000000000002) { + if (input[1] > 23.500000000000004) { + if (input[308] > 1e-35) { + var95 = 0.10511236013756364; + } else { + if (input[10] > 10.500000000000002) { + if (input[1] > 53.50000000000001) { + var95 = -0.08992396138178163; + } else { + var95 = 0.010944365997007212; + } + } else { + var95 = 0.06221307021813793; + } + } + } else { + var95 = 0.1286024087559141; + } + } else { + if (input[127] > 1e-35) { + var95 = 0.06568148624531012; + } else { + if (input[10] > 40.50000000000001) { + var95 = -0.07567979134643352; + } else { + if (input[5] > 5647.500000000001) { + var95 = 0.07594672895572069; + } else { + var95 = -0.018158016446439187; + } + } + } + } + } + } + } else { + if (input[6] > 55.50000000000001) { + var95 = 0.009293422430111872; + } else { + if (input[4] > 45.50000000000001) { + var95 = -0.017749818406964022; + } else { + if (input[2] > 46.50000000000001) { + var95 = 0.01714136511113982; + } else { + var95 = -0.0000724762291423549; + } + } + } + } + } + } + let var96: number; + if (input[1] > 24.500000000000004) { + if (input[103] > 1e-35) { + if (input[8] > 48.50000000000001) { + if (input[17] > 1e-35) { + var96 = -0.048689215588703864; + } else { + if (input[9] > 27.500000000000004) { + if (input[0] > 3916.5000000000005) { + var96 = 0.07084726276890757; + } else { + var96 = -0.11232323677722932; + } + } else { + var96 = 0.04812773089510436; + } + } + } else { + var96 = 0.11757502216780046; + } + } else { + if (input[5] > 1464.5000000000002) { + if (input[5] > 1505.5000000000002) { + if (input[167] > 1e-35) { + var96 = 0.07470606002425358; + } else { + if (input[1] > 53.50000000000001) { + if (input[132] > 1e-35) { + var96 = 0.0879462816013881; + } else { + var96 = -0.002966662093626573; + } + } else { + if (input[306] > 1e-35) { + var96 = -0.04588085188342676; + } else { + var96 = 0.0031910005157084823; + } + } + } + } else { + if (input[3] > 10.500000000000002) { + if (input[10] > 20.500000000000004) { + var96 = -0.006600332774461143; + } else { + var96 = 0.1272481351557754; + } + } else { + var96 = -0.09030973597154808; + } + } + } else { + if (input[284] > 1e-35) { + if (input[1] > 38.50000000000001) { + if (input[10] > 2.5000000000000004) { + var96 = 0.011884312066620044; + } else { + var96 = 0.11678751052403374; + } + } else { + if (input[4] > 8.500000000000002) { + var96 = 0.03627129613273813; + } else { + var96 = -0.12132783497902287; + } + } + } else { + var96 = -0.006784372643244717; + } + } + } + } else { + if (input[18] > 1e-35) { + if (input[3] > 4.500000000000001) { + if (input[3] > 6.500000000000001) { + if (input[0] > 5453.500000000001) { + var96 = -0.06830131718398992; + } else { + if (input[0] > 5147.500000000001) { + var96 = 0.062360406249609306; + } else { + if (input[4] > 4.500000000000001) { + var96 = -0.013162203864592055; + } else { + var96 = -0.07153029184927609; + } + } + } + } else { + var96 = 0.07628618062271557; + } + } else { + var96 = -0.12085065687320373; + } + } else { + if (input[190] > 1e-35) { + var96 = -0.045816889524231186; + } else { + if (input[137] > 1e-35) { + var96 = -0.07956001795911584; + } else { + if (input[199] > 1e-35) { + if (input[0] > 3853.5000000000005) { + var96 = 0.025895337822752502; + } else { + var96 = -0.06503949350616421; + } + } else { + if (input[10] > 227.50000000000003) { + var96 = -0.09989456525790491; + } else { + if (input[10] > 130.50000000000003) { + var96 = 0.08616651057030683; + } else { + var96 = 0.0001234981796706021; + } + } + } + } + } + } + } + let var97: number; + if (input[8] > 1014.5000000000001) { + if (input[9] > 137.50000000000003) { + var97 = -0.08778879924617534; + } else { + if (input[8] > 1022.5000000000001) { + if (input[285] > 1e-35) { + if (input[9] > 64.50000000000001) { + var97 = 0.04955806187281689; + } else { + if (input[0] > 3670.5000000000005) { + if (input[10] > 32.50000000000001) { + var97 = -0.141732381961068; + } else { + var97 = -0.0317152307496497; + } + } else { + var97 = -0.02074638849097191; + } + } + } else { + if (input[0] > 93.50000000000001) { + if (input[0] > 3072.5000000000005) { + if (input[10] > 100.50000000000001) { + if (input[4] > 24.500000000000004) { + if (input[8] > 1336.5000000000002) { + var97 = 0.12191801556691254; + } else { + var97 = -0.0003444689085397977; + } + } else { + var97 = 0.005739668504631604; + } + } else { + if (input[146] > 1e-35) { + if (input[308] > 1e-35) { + var97 = 0.015237524791728777; + } else { + if (input[6] > 61.50000000000001) { + if (input[4] > 63.50000000000001) { + var97 = -0.05676033995381961; + } else { + var97 = 0.10933961076803381; + } + } else { + if (input[4] > 26.500000000000004) { + var97 = -0.11667582544549814; + } else { + if (input[8] > 1765.5000000000002) { + var97 = 0.032174455312047705; + } else { + var97 = -0.0755016390126608; + } + } + } + } + } else { + if (input[293] > 1e-35) { + var97 = -0.08234885407658332; + } else { + if (input[9] > 41.50000000000001) { + if (input[0] > 3830.5000000000005) { + var97 = 0.026571311956824436; + } else { + if (input[15] > 1e-35) { + var97 = 0.06175459479851121; + } else { + var97 = -0.018778084411148754; + } + } + } else { + if (input[9] > 40.50000000000001) { + var97 = -0.09420232889965811; + } else { + var97 = -0.004578248021263184; + } + } + } + } + } + } else { + if (input[2] > 1.5000000000000002) { + var97 = 0.005453714644971445; + } else { + var97 = -0.03907138175699279; + } + } + } else { + var97 = -0.055296364182154736; + } + } + } else { + if (input[23] > 1e-35) { + var97 = 0.036555134842143476; + } else { + if (input[0] > 4188.500000000001) { + if (input[6] > 29.500000000000004) { + var97 = -0.09358146510580179; + } else { + var97 = 0.060524657996178094; + } + } else { + var97 = -0.11245101144669545; + } + } + } + } + } else { + if (input[125] > 1e-35) { + if (input[9] > 1.5000000000000002) { + var97 = -0.12698331085931538; + } else { + var97 = 0.006059605604079918; + } + } else { + if (input[2] > 196.50000000000003) { + var97 = -0.09451315810804783; + } else { + var97 = 0.0011390147031687425; + } + } + } + let var98: number; + if (input[8] > 2830.5000000000005) { + if (input[1] > 31.500000000000004) { + if (input[9] > 32.50000000000001) { + if (input[5] > 1234.5000000000002) { + if (input[8] > 3794.5000000000005) { + var98 = 0.05517359070460923; + } else { + var98 = -0.04758751221404857; + } + } else { + var98 = -0.09482078194138792; + } + } else { + if (input[8] > 2992.5000000000005) { + if (input[1] > 101.50000000000001) { + var98 = 0.1040436595565776; + } else { + if (input[9] > 21.500000000000004) { + var98 = 0.04032250517675179; + } else { + if (input[107] > 1e-35) { + var98 = 0.05978752253058374; + } else { + if (input[210] > 1e-35) { + if (input[4] > 37.50000000000001) { + var98 = 0.1192453009230486; + } else { + if (input[1] > 51.50000000000001) { + var98 = 0.0443376336292195; + } else { + var98 = -0.07967674833321865; + } + } + } else { + if (input[5] > 2117.5000000000005) { + if (input[9] > 10.500000000000002) { + var98 = -0.10025078607591283; + } else { + if (input[0] > 2882.5000000000005) { + if (input[18] > 1e-35) { + var98 = -0.08999822408398037; + } else { + var98 = 0.017533219253893447; + } + } else { + if (input[9] > 1.5000000000000002) { + if (input[4] > 12.500000000000002) { + var98 = -0.061850439226075; + } else { + var98 = 0.08849196353361093; + } + } else { + var98 = 0.10536348167793089; + } + } + } + } else { + if (input[92] > 1e-35) { + var98 = 0.04894947712119185; + } else { + if (input[9] > 16.500000000000004) { + var98 = 0.05900227903883853; + } else { + if (input[9] > 5.500000000000001) { + var98 = -0.11946594348916476; + } else { + var98 = -0.03652096348071964; + } + } + } + } + } + } + } + } + } else { + if (input[1] > 41.50000000000001) { + var98 = -0.07411603110840567; + } else { + var98 = -0.00021033247574340914; + } + } + } + } else { + if (input[10] > 22.500000000000004) { + if (input[9] > 68.50000000000001) { + var98 = 0.08493634342741495; + } else { + if (input[11] > 1e-35) { + var98 = -0.10899097825564363; + } else { + var98 = -0.006156708838964173; + } + } + } else { + if (input[8] > 3198.5000000000005) { + if (input[2] > 41.50000000000001) { + var98 = 0.08356655906359918; + } else { + if (input[7] > 25.500000000000004) { + var98 = -0.09475076526194888; + } else { + if (input[10] > 5.500000000000001) { + var98 = -0.01999406228763778; + } else { + var98 = 0.06696212545889428; + } + } + } + } else { + if (input[6] > 20.500000000000004) { + var98 = 0.14713592661393468; + } else { + var98 = 0.0459917279002218; + } + } + } + } + } else { + var98 = 0.00027445928493734093; + } + let var99: number; + if (input[223] > 1e-35) { + if (input[1] > 31.500000000000004) { + if (input[8] > 634.5000000000001) { + var99 = -0.06904501553217077; + } else { + var99 = 0.05696231672035904; + } + } else { + var99 = -0.1124703178077813; + } + } else { + if (input[99] > 1e-35) { + if (input[1] > 89.50000000000001) { + var99 = -0.05074261170009721; + } else { + if (input[1] > 57.50000000000001) { + if (input[8] > 969.5000000000001) { + var99 = -0.011419256378538392; + } else { + if (input[0] > 3830.5000000000005) { + var99 = 0.140315841503076; + } else { + var99 = 0.02403434913963024; + } + } + } else { + if (input[1] > 31.500000000000004) { + if (input[8] > 65.50000000000001) { + if (input[2] > 10.500000000000002) { + var99 = -0.04027822909411164; + } else { + var99 = 0.03176085103667189; + } + } else { + var99 = 0.06779515865838849; + } + } else { + if (input[4] > 15.500000000000002) { + var99 = 0.0762878389015175; + } else { + if (input[8] > 175.50000000000003) { + if (input[0] > 3030.5000000000005) { + if (input[8] > 1041.5000000000002) { + var99 = 0.06124039747298539; + } else { + var99 = -0.04312732764434027; + } + } else { + var99 = 0.09161522761808062; + } + } else { + var99 = -0.09663512235460074; + } + } + } + } + } + } else { + if (input[280] > 1e-35) { + if (input[6] > 45.50000000000001) { + if (input[1] > 46.50000000000001) { + var99 = 0.11211681010488772; + } else { + if (input[13] > 1e-35) { + var99 = 0.06725735814960367; + } else { + var99 = -0.046744031455827846; + } + } + } else { + if (input[10] > 44.50000000000001) { + if (input[0] > 3400.5000000000005) { + if (input[0] > 4004.5000000000005) { + if (input[2] > 22.500000000000004) { + var99 = 0.11743605068905603; + } else { + var99 = -0.011309033539148687; + } + } else { + var99 = -0.07896094707523052; + } + } else { + var99 = 0.12862714793172117; + } + } else { + if (input[10] > 1.5000000000000002) { + if (input[8] > 455.50000000000006) { + if (input[0] > 4706.500000000001) { + var99 = -0.09218756798869711; + } else { + if (input[10] > 19.500000000000004) { + if (input[0] > 1894.5000000000002) { + if (input[0] > 3719.5000000000005) { + var99 = 0.02836295848998302; + } else { + var99 = 0.12210680366745175; + } + } else { + var99 = -0.058302317470509096; + } + } else { + if (input[5] > 4144.500000000001) { + var99 = 0.06123341960495106; + } else { + var99 = -0.03840046906926525; + } + } + } + } else { + var99 = -0.05221474543453495; + } + } else { + var99 = 0.03988215485860711; + } + } + } + } else { + var99 = -0.00033074684693083496; + } + } + } + const var100: number = sigmoid( + var0 + + var1 + + var2 + + var3 + + var4 + + var5 + + var6 + + var7 + + var8 + + var9 + + var10 + + var11 + + var12 + + var13 + + var14 + + var15 + + var16 + + var17 + + var18 + + var19 + + var20 + + var21 + + var22 + + var23 + + var24 + + var25 + + var26 + + var27 + + var28 + + var29 + + var30 + + var31 + + var32 + + var33 + + var34 + + var35 + + var36 + + var37 + + var38 + + var39 + + var40 + + var41 + + var42 + + var43 + + var44 + + var45 + + var46 + + var47 + + var48 + + var49 + + var50 + + var51 + + var52 + + var53 + + var54 + + var55 + + var56 + + var57 + + var58 + + var59 + + var60 + + var61 + + var62 + + var63 + + var64 + + var65 + + var66 + + var67 + + var68 + + var69 + + var70 + + var71 + + var72 + + var73 + + var74 + + var75 + + var76 + + var77 + + var78 + + var79 + + var80 + + var81 + + var82 + + var83 + + var84 + + var85 + + var86 + + var87 + + var88 + + var89 + + var90 + + var91 + + var92 + + var93 + + var94 + + var95 + + var96 + + var97 + + var98 + + var99 + ); + return [1.0 - var100, var100]; +} +function sigmoid(x: number): number { + if (x < 0.0) { + const z: number = Math.exp(x); + return z / (1.0 + z); + } + return 1.0 / (1.0 + Math.exp(-x)); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/normalizeIndent.ts b/completions-sample-code/vscode-node/lib/src/ghostText/normalizeIndent.ts new file mode 100644 index 0000000..e9e1630 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/normalizeIndent.ts @@ -0,0 +1,71 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { GhostCompletion } from './ghostText'; + +export interface ITextEditorOptions { + tabSize?: number | string; + insertSpaces?: boolean | string; +} + +export function normalizeIndentCharacter( + options: ITextEditorOptions, + completion: GhostCompletion, + isEmptyLine: boolean +): GhostCompletion { + function replace(text: string, toReplace: string, replacer: (numberOfRemovedChars: number) => string): string { + const regex = new RegExp(`^(${toReplace})+`, 'g'); + + return text + .split('\n') + .map(line => { + const trimmed = line.replace(regex, ''); + const removedCharacters = line.length - trimmed.length; + return replacer(removedCharacters) + trimmed; + }) + .join('\n'); + } + + //Get the "size" of indentation + let indentSize: number; + if (options.tabSize === undefined || typeof options.tabSize === 'string') { + //Undefined or string case never happens when getting the indent size. This case is just for making TS typechecker happy. + indentSize = 4; + } else { + indentSize = options.tabSize; + } + + //If editor indentation is set to tabs + if (options.insertSpaces === false) { + const r = (txt: string) => + replace(txt, ' ', n => '\t'.repeat(Math.floor(n / indentSize)) + ' '.repeat(n % indentSize)); + completion.displayText = r(completion.displayText); + completion.completionText = r(completion.completionText); + } + //If editor indentation is set to spaces + else if (options.insertSpaces === true) { + const r = (txt: string) => replace(txt, '\t', n => ' '.repeat(n * indentSize)); + completion.displayText = r(completion.displayText); + completion.completionText = r(completion.completionText); + if (isEmptyLine) { + const re = (txt: string) => { + if (txt === '') { + return txt; + } + const firstLine = txt.split('\n')[0]; + const spacesAtStart = firstLine.length - firstLine.trimStart().length; + const remainder = spacesAtStart % indentSize; + if (remainder !== 0 && spacesAtStart > 0) { + const toReplace = ' '.repeat(remainder); + return replace(txt, toReplace, n => ' '.repeat((Math.floor(n / indentSize) + 1) * indentSize)); + } else { return txt; } + }; + + completion.displayText = re(completion.displayText); + completion.completionText = re(completion.completionText); + } + } + + return completion; +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/speculativeRequestCache.ts b/completions-sample-code/vscode-node/lib/src/ghostText/speculativeRequestCache.ts new file mode 100644 index 0000000..a57b04e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/speculativeRequestCache.ts @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { LRUCacheMap } from '../helpers/cache'; + +type RequestFunction = () => Promise<unknown>; + +export const ICompletionsSpeculativeRequestCache = createServiceIdentifier<ICompletionsSpeculativeRequestCache>('ICompletionsSpeculativeRequestCache'); +export interface ICompletionsSpeculativeRequestCache { + readonly _serviceBrand: undefined; + + set(completionId: string, requestFunction: RequestFunction): void; + request(completionId: string): Promise<void>; +} + +export class SpeculativeRequestCache implements ICompletionsSpeculativeRequestCache { + readonly _serviceBrand: undefined; + + private cache = new LRUCacheMap<string, RequestFunction>(100); + + set(completionId: string, requestFunction: RequestFunction): void { + this.cache.set(completionId, requestFunction); + } + + async request(completionId: string): Promise<void> { + const fn = this.cache.get(completionId); + if (fn === undefined) { return; } + this.cache.delete(completionId); + await fn(); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/statementTree.ts b/completions-sample-code/vscode-node/lib/src/ghostText/statementTree.ts new file mode 100644 index 0000000..06f8099 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/statementTree.ts @@ -0,0 +1,833 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import Parser, { SyntaxNode } from 'web-tree-sitter'; +import { parseTreeSitter } from '../../../prompt/src/parse'; + +export abstract class StatementNode { + readonly children: StatementNode[] = []; + parent: StatementNode | undefined; + nextSibling: StatementNode | undefined; + protected collapsed = false; + + constructor(readonly node: SyntaxNode) { } + + addChild(child: StatementNode) { + child.parent = this; + child.nextSibling = undefined; + if (this.children.length > 0) { + this.children[this.children.length - 1].nextSibling = child; + } + this.children.push(child); + } + + /** + * Called after the last child is added to this node when the tree is being + * constructed. This is a callback derived classes can use to do any additional + * processing once this branch of the tree is complete. The default behavior + * is to do nothing. + */ + childrenFinished() { } + + containsStatement(stmt: StatementNode): boolean { + return this.node.startIndex <= stmt.node.startIndex && this.node.endIndex >= stmt.node.endIndex; + } + + statementAt(offset: number): StatementNode | undefined { + if (this.node.startIndex > offset || this.node.endIndex < offset) { return undefined; } + + let innerMatch: StatementNode | undefined = undefined; + this.children.find(stmt => { + innerMatch = stmt.statementAt(offset); + return innerMatch !== undefined; + }); + return innerMatch ?? this; + } + + abstract get isCompoundStatementType(): boolean; + + /** Treat this node and its children as a single statement */ + protected collapse() { + this.children.length = 0; + this.collapsed = true; + } + + get description(): string { + return `${this.node.type} ([${this.node.startPosition.row},${this.node.startPosition.column}]..[${this.node.endPosition.row},${this.node.endPosition.column}]): ${JSON.stringify(this.node.text.length > 33 ? this.node.text.substring(0, 15) + '...' + this.node.text.slice(-15) : this.node.text)}`; + } + + dump(prefix1: string = '', prefix2: string = ''): string { + const result = [`${prefix1}${this.description}`]; + this.children.forEach(child => { + result.push( + child.dump(`${prefix2}+- `, child.nextSibling === undefined ? `${prefix2} ` : `${prefix2}| `) + ); + }); + return result.join('\n'); + } + + dumpPath(prefix1: string = '', prefix2: string = '', forChild = false): string { + if (this.parent) { + const path = this.parent.dumpPath(prefix1, prefix2, true); + const indentSize = path.length - path.lastIndexOf('\n') - 1 - prefix2.length; + const indent = ' '.repeat(indentSize); + const nextPrefix = forChild ? `\n${prefix2}${indent}+- ` : ''; + return path + this.description + nextPrefix; + } else { + const nextPrefix = forChild ? `\n${prefix2}+- ` : ''; + return prefix1 + this.description + nextPrefix; + } + } +} + +/** + * A simplified view of a syntax tree. + * + * It contains only nodes which represent complete statements. Because statements may + * be compound, a single statement may contain other statements within it. + * + * "Statement" refers to a syntactic unit of the language. It represents the smallest + * division of a code completion we would consider when truncating. It may be a simple + * statement such as: + * + * `x = 1;` + * + * or a compound statement such as: + * + * `if (x > 0) { x = 2; }` + * + * where the entire string comprises the parent statement, and `x = 2;` is + * a child statement of the parent. Note that `x > 0` is not a statement, but + * an expression. + * + * The view is further constrained to a portion of the overall document (the start and + * end offsets). This view contains all statements which intersect that region, so + * containing statements are included in the view, even though they may extend + * beyond the region. + */ +export abstract class StatementTree implements Disposable { + protected tree: Parser.Tree | undefined; + readonly statements: StatementNode[] = []; + + static isSupported(languageId: string): boolean { + return ( + JSStatementTree.languageIds.has(languageId) || + TSStatementTree.languageIds.has(languageId) || + PyStatementTree.languageIds.has(languageId) || + GoStatementTree.languageIds.has(languageId) || + PhpStatementTree.languageIds.has(languageId) || + RubyStatementTree.languageIds.has(languageId) || + JavaStatementTree.languageIds.has(languageId) || + CSharpStatementTree.languageIds.has(languageId) || + CStatementTree.languageIds.has(languageId) + ); + } + + static isTrimmedByDefault(languageId: string): boolean { + return ( + JSStatementTree.languageIds.has(languageId) || + TSStatementTree.languageIds.has(languageId) || + GoStatementTree.languageIds.has(languageId) + ); + } + + static create(languageId: string, text: string, startOffset: number, endOffset: number): StatementTree { + if (JSStatementTree.languageIds.has(languageId)) { + return new JSStatementTree(languageId, text, startOffset, endOffset); + } else if (TSStatementTree.languageIds.has(languageId)) { + return new TSStatementTree(languageId, text, startOffset, endOffset); + } else if (PyStatementTree.languageIds.has(languageId)) { + return new PyStatementTree(languageId, text, startOffset, endOffset); + } else if (GoStatementTree.languageIds.has(languageId)) { + return new GoStatementTree(languageId, text, startOffset, endOffset); + } else if (JavaStatementTree.languageIds.has(languageId)) { + return new JavaStatementTree(languageId, text, startOffset, endOffset); + } else if (PhpStatementTree.languageIds.has(languageId)) { + return new PhpStatementTree(languageId, text, startOffset, endOffset); + } else if (RubyStatementTree.languageIds.has(languageId)) { + return new RubyStatementTree(languageId, text, startOffset, endOffset); + } else if (CSharpStatementTree.languageIds.has(languageId)) { + return new CSharpStatementTree(languageId, text, startOffset, endOffset); + } else if (CStatementTree.languageIds.has(languageId)) { + return new CStatementTree(languageId, text, startOffset, endOffset); + } else { + throw new Error(`Unsupported languageId: ${languageId}`); + } + } + + constructor( + private readonly languageId: string, + private readonly text: string, + private readonly startOffset: number, + private readonly endOffset: number + ) { } + + [Symbol.dispose]() { + if (this.tree) { + this.tree.delete(); + this.tree = undefined; + } + } + + clear() { + this.statements.length = 0; + } + + statementAt(offset: number): StatementNode | undefined { + let match: StatementNode | undefined = undefined; + this.statements.find(stmt => { + match = stmt.statementAt(offset); + return match !== undefined; + }); + return match; + } + + async build(): Promise<void> { + const parents: StatementNode[] = []; + this.clear(); + const tree = await this.parse(); + const query = this.getStatementQuery(tree); + query + .captures(tree.rootNode, { + startPosition: this.offsetToPosition(this.startOffset), + endPosition: this.offsetToPosition(this.endOffset), + }) + .forEach(capture => { + const stmt = this.createNode(capture.node); + while (parents.length > 0 && !parents[0].containsStatement(stmt)) { + const completed = parents.shift(); // not a parent + completed?.childrenFinished(); + } + if (parents.length > 0) { + parents[0].addChild(stmt); // this is our parent + } else { + this.addStatement(stmt); // top-level statement + } + parents.unshift(stmt); // add to the stack + }); + // finish up + parents.forEach(stmt => stmt.childrenFinished()); + } + + protected abstract createNode(node: SyntaxNode): StatementNode; + protected abstract getStatementQueryText(): string; + + protected addStatement(stmt: StatementNode) { + stmt.parent = undefined; + stmt.nextSibling = undefined; + if (this.statements.length > 0) { + this.statements[this.statements.length - 1].nextSibling = stmt; + } + this.statements.push(stmt); + } + + protected async parse(): Promise<Parser.Tree> { + if (!this.tree) { + this.tree = await parseTreeSitter(this.languageId, this.text); + } + return this.tree; + } + + protected getStatementQuery(tree: Parser.Tree): Parser.Query { + return this.getQuery(tree.getLanguage(), this.getStatementQueryText()); + } + + protected getQuery(language: Parser.Language, queryText: string): Parser.Query { + // TODO: query objects can be cached and reused + return language.query(queryText); + } + + protected offsetToPosition(offset: number): Parser.Point { + const lines = this.text.slice(0, offset).split('\n'); + const row = lines.length - 1; + const column = lines[lines.length - 1].length; + return { row, column }; + } + + dump(prefix: string = ''): string { + const result: string[] = []; + this.statements.forEach((stmt, idx) => { + const idxStr = `[${idx}]`; + const idxSpaces = ' '.repeat(idxStr.length); + result.push(stmt.dump(`${prefix} ${idxStr} `, `${prefix} ${idxSpaces} `)); + }); + return result.join('\n'); + } +} + +/* + * Javascript and Typescript implementation + */ + +class JSStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'function_declaration', + 'generator_function_declaration', + 'class_declaration', + 'statement_block', + 'if_statement', + 'switch_statement', + 'for_statement', + 'for_in_statement', + 'while_statement', + 'do_statement', + 'try_statement', + 'with_statement', + 'labeled_statement', + 'method_definition', + 'interface_declaration', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && JSStatementNode.compoundTypeNames.has(this.node.type); + } + + override childrenFinished() { + if (this.isSingleLineIfStatement()) { this.collapse(); } + } + + private isSingleLineIfStatement(): boolean { + // must be an if statement + if (this.node.type !== 'if_statement') { return false; } + // must be a single line + if (this.node.startPosition.row !== this.node.endPosition.row) { return false; } + // Exclude if statements with braces so that block position is correct: + // can have a single statement without braces + if (this.children.length === 1 && this.children[0].node.type !== 'statement_block') { return true; } + // or two statements without braces if an else is present + if ( + this.children.length === 2 && + this.node.childForFieldName('alternative') !== null && + this.children[0].node.type !== 'statement_block' && + this.children[1].node.type !== 'statement_block' + ) { + return true; + } + + return false; + } +} + +class JSStatementTree extends StatementTree { + static readonly languageIds = new Set(['javascript', 'javascriptreact', 'jsx']); + + protected createNode(node: SyntaxNode): StatementNode { + return new JSStatementNode(node); + } + + protected getStatementQueryText(): string { + // From https://github.com/tree-sitter/tree-sitter-javascript/blob/fdeb68ac8d2bd5a78b943528bb68ceda3aade2eb/grammar.js#L199-L226 + // Because `statement` is declared `inline` in this version of the + // grammar, we search for each choice from its definition plus two + // class constructs we want to consider for trimming. + return `[ + (export_statement) + (import_statement) + (debugger_statement) + (expression_statement) + (declaration) + (statement_block) + (if_statement) + (switch_statement) + (for_statement) + (for_in_statement) + (while_statement) + (do_statement) + (try_statement) + (with_statement) + (break_statement) + (continue_statement) + (return_statement) + (throw_statement) + (empty_statement) + (labeled_statement) + (method_definition) + (field_definition) + ] @statement`; + } +} + +class TSStatementTree extends StatementTree { + static readonly languageIds = new Set(['typescript', 'typescriptreact']); + + protected createNode(node: SyntaxNode): StatementNode { + return new JSStatementNode(node); + } + + protected getStatementQueryText(): string { + // From https://github.com/tree-sitter/tree-sitter-javascript/blob/fdeb68ac8d2bd5a78b943528bb68ceda3aade2eb/grammar.js#L199-L226 + // Because `statement` is declared `inline` in this version of the + // grammar, we search for each choice from its definition plus two + // class constructs we want to consider for trimming. + return `[ + (export_statement) + (import_statement) + (debugger_statement) + (expression_statement) + (declaration) + (statement_block) + (if_statement) + (switch_statement) + (for_statement) + (for_in_statement) + (while_statement) + (do_statement) + (try_statement) + (with_statement) + (break_statement) + (continue_statement) + (return_statement) + (throw_statement) + (empty_statement) + (labeled_statement) + (method_definition) + (public_field_definition) + ] @statement`; + } +} + +/* + * Python implementation + */ +class PyStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'if_statement', + 'for_statement', + 'while_statement', + 'try_statement', + 'with_statement', + 'function_definition', + 'class_definition', + 'decorated_definition', + 'match_statement', + 'block', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && PyStatementNode.compoundTypeNames.has(this.node.type); + } + + override childrenFinished() { + if (this.isSingleLineIfStatement()) { this.collapse(); } + } + + private isSingleLineIfStatement(): boolean { + // must be an if statement + if (this.node.type !== 'if_statement') { return false; } + // must be a single line + return this.node.startPosition.row === this.node.endPosition.row; + } +} + +class PyStatementTree extends StatementTree { + static readonly languageIds = new Set(['python']); + + protected createNode(node: SyntaxNode): StatementNode { + return new PyStatementNode(node); + } + + protected getStatementQueryText(): string { + // Search for nodes of type `_simple_statement` and `_compound_statement`. + // Because these are both inlined, we search for each choice in the two + // definitions. It also adds `block` to more closely match the tree + // shape of JS/TS. + // + // For the _simple_statement definition see: https://github.com/tree-sitter/tree-sitter-python/blob/7473026494597de8bc403735b1bfec7ca846c0d6/grammar.js#L90-L106 + // For the _compound_statement definition see: https://github.com/tree-sitter/tree-sitter-python/blob/7473026494597de8bc403735b1bfec7ca846c0d6/grammar.js#L230-L240 + return `[ + (future_import_statement) + (import_statement) + (import_from_statement) + (print_statement) + (assert_statement) + (expression_statement) + (return_statement) + (delete_statement) + (raise_statement) + (pass_statement) + (break_statement) + (continue_statement) + (global_statement) + (nonlocal_statement) + (exec_statement) + (if_statement) + (for_statement) + (while_statement) + (try_statement) + (with_statement) + (function_definition) + (class_definition) + (decorated_definition) + (match_statement) + (block) + ] @statement`; + } +} + +/* + * Go implementation + */ +class GoStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'function_declaration', + 'method_declaration', + 'if_statement', + 'for_statement', + 'expression_switch_statement', + 'type_switch_statement', + 'select_statement', + 'block', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && GoStatementNode.compoundTypeNames.has(this.node.type); + } +} + +class GoStatementTree extends StatementTree { + static readonly languageIds = new Set(['go']); + + protected createNode(node: SyntaxNode): StatementNode { + return new GoStatementNode(node); + } + + protected getStatementQueryText(): string { + // Search for nodes of type `_top_level_declaration` and `_statement`. + // Because `_top_level_declaration` is inlined, we search for each + // choice in its definition. It also adds `block` to match the tree + // shape of JS/TS. + // + // For the _top_level_declaration definition see: https://github.com/tree-sitter/tree-sitter-go/blob/3c3775faa968158a8b4ac190a7fda867fd5fb748/grammar.js#L117-L122 + return `[ + (package_clause) + (function_declaration) + (method_declaration) + (import_declaration) + (_statement) + (block) + ] @statement`; + } +} + +/** + * Php implementation + */ +class PhpStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'if_statement', + 'else_clause', + 'else_if_clause', + 'for_statement', + 'foreach_statement', + 'while_statement', + 'do_statement', + 'switch_statement', + 'try_statement', + 'catch_clause', + 'finally_clause', + 'anonymous_function', + 'compound_statement', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && PhpStatementNode.compoundTypeNames.has(this.node.type); + } +} + +class PhpStatementTree extends StatementTree { + static readonly languageIds = new Set(['php']); + + protected override createNode(node: SyntaxNode): StatementNode { + return new PhpStatementNode(node); + } + protected override getStatementQueryText(): string { + // Search for nodes of type `_statement` and a few other picked types. + // `compound_statement`, `method_declaration`, `property_declaration`, `const_declaration`, and `use_declaration` are + // not encompassed by `_statement` so we add them to the query to make multi-line reveal more useful. + // For the _statement definition see: https://github.com/tree-sitter/tree-sitter-php/blob/eb289f127fc341ae7129902a2dd1c6c197a4c1e7/common/define-grammar.js#L141 + return `[ + (statement) + (compound_statement) + (method_declaration) + (property_declaration) + (const_declaration) + (use_declaration) + ] @statement`; + } +} + +/** + * Ruby implementation + */ + +class RubyStatementNode extends StatementNode { + static compoundTypeNames = new Set(['if', 'case', 'while', 'until', 'for', 'begin', 'module', 'class', 'method']); + + get isCompoundStatementType(): boolean { + return !this.collapsed && RubyStatementNode.compoundTypeNames.has(this.node.type); + } +} + +class RubyStatementTree extends StatementTree { + static readonly languageIds = new Set(['ruby']); + + protected createNode(node: SyntaxNode): StatementNode { + return new RubyStatementNode(node); + } + //(if_modifier) + protected getStatementQueryText(): string { + return `[ + (_statement) + (when) + ] @statement`; + } +} + +/* + * Java implementation + */ + +class JavaStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'block', + 'do_statement', + 'enhanced_for_statement', + 'for_statement', + 'if_statement', + 'labeled_statement', + 'switch_expression', + 'synchronized_statement', + 'try_statement', + 'try_with_resources_statement', + 'while_statement', + 'interface_declaration', + 'method_declaration', + 'constructor_declaration', + 'compact_constructor_declaration', + 'class_declaration', + 'annotation_type_declaration', + 'static_initializer', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && JavaStatementNode.compoundTypeNames.has(this.node.type); + } + + override childrenFinished() { + // Collapse if_statements on a single line + if (this.isSingleLineIfStatement()) { this.collapse(); } + } + + private isSingleLineIfStatement(): boolean { + // must be an if statement + if (this.node.type !== 'if_statement') { return false; } + // must be a single line + if (this.node.startPosition.row !== this.node.endPosition.row) { return false; } + // Exclude if statements with braces so that block position is correct: + // can have a single statement without braces + if (this.children.length === 1 && this.children[0].node.type !== 'block') { return true; } + + return false; + } +} + +class JavaStatementTree extends StatementTree { + // Grammar via: https://github.com/tree-sitter/tree-sitter-java/blob/master/src/grammar.json + // Node types via: https://github.com/tree-sitter/tree-sitter-java/blob/master/src/node-types.json + static readonly languageIds = new Set(['java']); + + protected createNode(node: SyntaxNode): StatementNode { + return new JavaStatementNode(node); + } + + // _class_body_declaration is inlined, so add those subtypes to the query + protected getStatementQueryText(): string { + return `[ + (statement) + (field_declaration) + (record_declaration) + (method_declaration) + (compact_constructor_declaration) + (class_declaration) + (interface_declaration) + (annotation_type_declaration) + (enum_declaration) + (block) + (static_initializer) + (constructor_declaration) + ] @statement`; + } +} + +/* + * C# implementation + */ +class CSharpStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'block', + 'checked_statement', + 'class_declaration', + 'constructor_declaration', + 'destructor_declaration', + 'do_statement', + 'fixed_statement', + 'for_statement', + 'foreach_statement', + 'if_statement', + 'interface_declaration', + 'lock_statement', + 'method_declaration', + 'struct_declaration', + 'switch_statement', + 'try_statement', + 'unsafe_statement', + 'while_statement', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && CSharpStatementNode.compoundTypeNames.has(this.node.type); + } + + override childrenFinished() { + if (this.isSingleLineIfStatement()) { this.collapse(); } + } + + private isSingleLineIfStatement(): boolean { + // must be an if statement + if (this.node.type !== 'if_statement') { return false; } + // must be a single line + if (this.node.startPosition.row !== this.node.endPosition.row) { return false; } + // Exclude if statements with braces so that block position is correct: + // can have a single statement without braces + if (this.children.length === 1 && this.children[0].node.type !== 'block') { return true; } + + return false; + } +} + +class CSharpStatementTree extends StatementTree { + static readonly languageIds = new Set(['csharp']); + + protected createNode(node: SyntaxNode): StatementNode { + return new CSharpStatementNode(node); + } + + protected getStatementQueryText(): string { + return `[ + (extern_alias_directive) + (using_directive) + (global_attribute) + (preproc_if) + (namespace_declaration) + (file_scoped_namespace_declaration) + (statement) + (type_declaration) + (declaration) + (accessor_declaration) + (block) + ] @statement`; + } +} + +/* + * C, C++ implementation + */ + +class CStatementNode extends StatementNode { + static compoundTypeNames = new Set([ + 'declaration', + 'function_definition', + 'enum_specifier', + 'field_declaration_list', + 'type_definition', + 'compound_statement', + 'if_statement', + 'switch_statement', + 'while_statement', + 'for_statement', + 'do_statement', + 'preproc_if', + 'preproc_ifdef', + + // C++ specific: + 'namespace_definition', + 'class_specifier', + 'field_declaration_list', + 'concept_definition', + 'template_declaration', + ]); + + get isCompoundStatementType(): boolean { + return !this.collapsed && CStatementNode.compoundTypeNames.has(this.node.type); + } + + override childrenFinished() { + if (this.isSingleLineDeclarationStatement() || this.isSingleLineConceptDefinition()) { this.collapse(); } + } + + private isSingleLineDeclarationStatement(): boolean { + // must be an declaration statement + if (this.node.type !== 'declaration') { return false; } + // must be a single line + if (this.node.startPosition.row !== this.node.endPosition.row) { return false; } + return true; + } + + private isSingleLineConceptDefinition(): boolean { + // must be a concept definition + if (this.node.type !== 'concept_definition') { return false; } + // must be a single line + if (this.node.startPosition.row !== this.node.endPosition.row) { return false; } + return true; + } +} + +class CStatementTree extends StatementTree { + static readonly languageIds = new Set(['c', 'cpp']); + + protected createNode(node: SyntaxNode): StatementNode { + return new CStatementNode(node); + } + + protected getStatementQueryText(): string { + return `[ + (declaration) + (function_definition) + (type_definition) + (field_declaration) + (enum_specifier) + (return_statement) + (compound_statement) + (if_statement) + (expression_statement) + (switch_statement) + (break_statement) + (case_statement) + (while_statement) + (for_statement) + (do_statement) + (goto_statement) + (labeled_statement) + (preproc_if) + (preproc_def) + (preproc_ifdef) + (preproc_include) + (preproc_call) + (preproc_function_def) + (continue_statement) + + ;C++ specific: + (namespace_definition) + (class_specifier) + (field_declaration_list) + (field_declaration) + (concept_definition) + (compound_requirement) + (template_declaration) + (using_declaration) + (alias_declaration) + (static_assert_declaration) + ] @statement`; + } +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/streamedCompletionSplitter.ts b/completions-sample-code/vscode-node/lib/src/ghostText/streamedCompletionSplitter.ts new file mode 100644 index 0000000..ad2ad33 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/streamedCompletionSplitter.ts @@ -0,0 +1,231 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { FinishedCallback, RequestDelta, SolutionDecision } from '../openai/fetch'; +import { APIChoice, convertToAPIChoice } from '../openai/openai'; +import { TerseBlockTrimmer } from './blockTrimmer'; + +class StreamingCompletion { + startOffset = 0; + text = ''; + trimCount = 0; + + constructor( + readonly index: number, + readonly documentPrefix: string + ) { } + + updateText(text: string): void { + this.text = text; + } + + get addedToPrefix(): string { + return this.text.substring(0, this.startOffset); + } + + get effectivePrefix(): string { + return this.documentPrefix + this.addedToPrefix; + } + + get effectiveText(): string { + return this.text.substring(this.startOffset); + } + + get isFirstCompletion(): boolean { + return this.trimCount === 0; + } + + /** + * Returns the index of the line ending to use when trimming the completion + * as a "single line" completion. This allows the completion to begin with + * a single leading new line as a special case for completing the next line. + * It supports CRLF and LF line endings. The index is the start of the line + * terminator. Returns -1 if a suitable line ending was not found. + */ + get firstNewlineOffset(): number { + const matches = [...this.text.matchAll(/\r?\n/g)]; + if (matches.length > 0 && matches[0].index === 0) { + matches.shift(); + } + return matches.length > 0 ? matches[0].index : -1; + } + + trimAt(effectiveOffset: number): StreamingCompletion { + const trimmed = new StreamingCompletion(this.index, this.documentPrefix); + trimmed.startOffset = this.startOffset; + trimmed.text = this.text.substring(0, this.startOffset + effectiveOffset); + trimmed.trimCount = this.trimCount; + this.startOffset += effectiveOffset; + this.trimCount++; + return trimmed; + } +} + +export class StreamedCompletionSplitter { + private readonly lineLimit = 3; + private readonly completions = new Map<number, StreamingCompletion>(); + + constructor( + private readonly prefix: string, + private readonly languageId: string, + private readonly initialSingleLine: boolean, + private readonly trimmerLookahead: number, + private readonly cacheFunction: (prefixAddition: string, item: APIChoice) => void, + @IInstantiationService private readonly instantiationService: IInstantiationService, + ) { } + + getFinishedCallback(): FinishedCallback { + return async (completionText: string, delta: RequestDelta): Promise<SolutionDecision> => { + const index = delta.index ?? 0; + const completion = this.getCompletion(index, completionText); + + // emmulate single line completion when this.initialSingleLine is set + if (completion.isFirstCompletion && this.initialSingleLine && completion.firstNewlineOffset >= 0) { + const result = { + yieldSolution: true, + continueStreaming: true, + finishOffset: completion.firstNewlineOffset, + }; + completion.trimAt(result.finishOffset); + if (delta.finished) { + await this.trimAll(delta, completion); + } + return result; + } + + return delta.finished ? await this.trimAll(delta, completion) : await this.trimOnce(delta, completion); + }; + } + + private getCompletion(index: number, newText: string): StreamingCompletion { + let completion = this.completions.get(index); + if (!completion) { + completion = new StreamingCompletion(index, this.prefix); + this.completions.set(index, completion); + } + completion.updateText(newText); + return completion; + } + + private async trimOnce(delta: RequestDelta, completion: StreamingCompletion): Promise<SolutionDecision> { + const offset = await this.trim(completion); + if (offset === undefined) { + return { + yieldSolution: false, + continueStreaming: true, + }; + } + + if (completion.isFirstCompletion) { + completion.trimAt(offset); + return { + yieldSolution: true, + continueStreaming: true, + finishOffset: offset, + }; + } else { + this.cacheCompletion(delta, completion, offset); + return { + yieldSolution: false, + continueStreaming: true, + }; + } + } + + private async trimAll(delta: RequestDelta, completion: StreamingCompletion): Promise<SolutionDecision> { + let offset: number | undefined; + let firstOffset: number | undefined; + + do { + offset = await this.trim(completion); + + if (completion.isFirstCompletion) { + firstOffset = offset; + completion.trimAt(offset ?? completion.effectiveText.length); + } else { + this.cacheCompletion(delta, completion, offset); + } + } while (offset !== undefined); + + if (firstOffset !== undefined) { + return { + yieldSolution: true, + continueStreaming: true, + finishOffset: firstOffset, + }; + } + + return { + yieldSolution: false, + continueStreaming: true, + }; + } + + private async trim(completion: StreamingCompletion): Promise<number | undefined> { + const trimmer = new TerseBlockTrimmer( + this.languageId, + completion.effectivePrefix, + completion.effectiveText, + this.lineLimit, + this.trimmerLookahead + ); + return await trimmer.getCompletionTrimOffset(); + } + + private cacheCompletion(delta: RequestDelta, completion: StreamingCompletion, offset?: number) { + const trimmed = completion.trimAt(offset ?? completion.effectiveText.length); + if (trimmed.effectiveText.trim() === '') { + return; + } + const apiChoice = this.instantiationService.invokeFunction(convertToAPIChoice, + trimmed.effectiveText.trimEnd(), + delta.getAPIJsonData!(), // FIXME@ulugbekna + trimmed.index, + delta.requestId!, // FIXME@ulugbekna + offset !== undefined, + delta.telemetryData! + ); + apiChoice.copilotAnnotations = this.adjustedAnnotations(apiChoice, completion, trimmed); + apiChoice.generatedChoiceIndex = trimmed.trimCount; + + this.cacheFunction(trimmed.addedToPrefix, apiChoice); + } + + private adjustedAnnotations( + choice: APIChoice, + fullCompletion: StreamingCompletion, + trimmedCompletion: StreamingCompletion + ): CopilotNamedAnnotationList | undefined { + if (choice.copilotAnnotations === undefined) { return undefined; } + + const newStartOffset = trimmedCompletion.addedToPrefix.length; + const newEndOffset = newStartOffset + choice.completionText.length; + // whether the current split choice is at the end of the original choice + const atEnd = newEndOffset >= fullCompletion.text.length; + + const adjusted: CopilotNamedAnnotationList = {}; + for (const [name, annotationGroup] of Object.entries(choice.copilotAnnotations)) { + const adjustedAnnotations = annotationGroup + .filter(a => { + return ( + a.start_offset - newStartOffset < choice.completionText.length && + a.stop_offset - newStartOffset > 0 + ); + }) + .map(a => { + const newA = { ...a }; + newA.start_offset -= newStartOffset; + newA.stop_offset -= newStartOffset; + if (!atEnd) { newA.stop_offset = Math.min(newA.stop_offset, choice.completionText.length); } + return newA; + }); + if (adjustedAnnotations.length > 0) { + adjusted[name] = adjustedAnnotations; + } + } + return Object.keys(adjusted).length > 0 ? adjusted : undefined; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/telemetry.ts b/completions-sample-code/vscode-node/lib/src/ghostText/telemetry.ts new file mode 100644 index 0000000..3642c4c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/telemetry.ts @@ -0,0 +1,221 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { PromptResponse } from '../prompt/prompt'; +import { now, telemetry, TelemetryData, telemetryRaw, TelemetryWithExp } from '../telemetry'; +import { CopilotCompletion } from './copilotCompletion'; +import { ResultType } from './ghostText'; +import { ICompletionsSpeculativeRequestCache } from './speculativeRequestCache'; + +export type PostInsertionCategory = 'ghostText' | 'solution'; + +export const GHOST_TEXT_CATEGORY: PostInsertionCategory = 'ghostText'; + +export const logger = new Logger('getCompletions'); + +/** Send `.shown` event */ +export function telemetryShown(accessor: ServicesAccessor, completion: CopilotCompletion) { + const speculativeRequestCache = accessor.get(ICompletionsSpeculativeRequestCache); + void speculativeRequestCache.request(completion.clientCompletionId); + completion.telemetry.markAsDisplayed(); // TODO: Consider removing displayedTime as unused and generally incorrect. + completion.telemetry.properties.reason = resultTypeToString(completion.resultType); + telemetry(accessor, `ghostText.shown`, completion.telemetry); +} + +/** Send `.accepted` event */ +export function telemetryAccepted( + accessor: ServicesAccessor, + insertionCategory: PostInsertionCategory, + telemetryData: TelemetryData +) { + const telemetryName = insertionCategory + '.accepted'; + + telemetry(accessor, telemetryName, telemetryData); +} + +/** Send `.rejected` event */ +export function telemetryRejected( + accessor: ServicesAccessor, + insertionCategory: PostInsertionCategory, + telemetryData: TelemetryData +) { + const telemetryName = insertionCategory + '.rejected'; + + telemetry(accessor, telemetryName, telemetryData); +} + +/** Cut down telemetry type for "result" telemetry, to avoid too much data load on Azure Monitor. + * + */ +type BasicResultTelemetry = { + headerRequestId: string; + copilot_trackingId: string; + opportunityId?: string; + sku?: string; + organizations_list?: string; + enterprise_list?: string; + clientCompletionId?: string; +}; + +/** + * For `ghostText.canceled` we include all fields for backwards compatibility, as this event had it initially, + * Note that we now send the event from more places, but it still makes sense to be consistent. + */ +type CanceledResultTelemetry = { + telemetryBlob: TelemetryData; + cancelledNetworkRequest?: boolean; // omitted is equivalent to false +}; + +/** + * When we request ghost text, we also send a `ghostText.issued` telemetry event. To measure + * Copilot's overall reliability, we want to make sure we consistently send a matching "result" event. + * + * This type allows us to keep track of what happened during the pipeline that produces ghost text results, + * and use the TypeScript type system to reduce the chances of accidentally forgetting to send the result event. + * + * At the end of that pipeline, we will either have a final ghost text result and we can send a `ghostText.produced` + * message, or something will have prevented us producing a result and we can send an alternative mesages. + */ +export type GhostTextResultWithTelemetry<T> = + /** + * A result was produced successfully. If this is the final ghost text result, + * we should send the result message `ghostText.produced`. + */ + | { + type: 'success'; + value: T; + telemetryData: BasicResultTelemetry; + // This is needed to populate the telemetryBlob in `ghostText.canceled` if this happens later. + telemetryBlob: TelemetryWithExp; + resultType: ResultType; + performanceMetrics?: [string, number][]; + } + /** + * We decided not to request ghost text this time. No `ghostText.issued` message + * was sent so there is no need send any result telemetry. + */ + | { type: 'abortedBeforeIssued'; reason: string; telemetryData: BasicResultTelemetry } + /** + * We requested ghost text, but we decided to cancel mid-way, for example because the + * user kept typing. This will turn into a `ghostText.canceled` result message. + * Note: this uses the preferred American spelling "canceled" rather than "cancelled", + * because the telemetry message has always done that, even though it may be inconsistent + * with log messages and code comments etc. + */ + | { type: 'canceled'; reason: string; telemetryData: CanceledResultTelemetry } + /** + * We requested ghost text, but didn't come up with any results for some "expected" + * reason, such as slur redaction or snippy. This will turn into a `ghostText.empty` + * result message. + */ + | { type: 'empty'; reason: string; telemetryData: BasicResultTelemetry } + /** + * We requested ghost text, but didn't come up with any results because something + * unexpected went wrong. This will turn into a `ghostText.failed` result message. + */ + | { type: 'failed'; reason: string; telemetryData: BasicResultTelemetry } + /** + * The promptOnly parameter was set to true in the request. We only need the prompt + * that was about to be sent to the model. This is for experimentation purposes, so + * there is not any need for telemetry in this case. + */ + | { type: 'promptOnly'; reason: string; prompt: PromptResponse }; + +export function mkCanceledResultTelemetry( + telemetryBlob: TelemetryData, + extraFlags: { cancelledNetworkRequest?: boolean } = {} +): CanceledResultTelemetry { + return { + ...extraFlags, + telemetryBlob, + }; +} + +export function mkBasicResultTelemetry( + telemetryBlob: TelemetryWithExp, +): BasicResultTelemetry { + const result: BasicResultTelemetry = { + headerRequestId: telemetryBlob.properties['headerRequestId'], + copilot_trackingId: telemetryBlob.properties['copilot_trackingId'], + }; + // copy certain properties if present + if (telemetryBlob.properties['sku'] !== undefined) { + result.sku = telemetryBlob.properties['sku']; + } + if (telemetryBlob.properties['opportunityId'] !== undefined) { + result.opportunityId = telemetryBlob.properties['opportunityId']; + } + if (telemetryBlob.properties['organizations_list'] !== undefined) { + result.organizations_list = telemetryBlob.properties['organizations_list']; + } + if (telemetryBlob.properties['enterprise_list'] !== undefined) { + result.enterprise_list = telemetryBlob.properties['enterprise_list']; + } + if (telemetryBlob.properties['clientCompletionId'] !== undefined) { + result.clientCompletionId = telemetryBlob.properties['clientCompletionId']; + } + + return result; +} + +/** + * Given a ghost text result, send the appropriate "result" telemetry, if any, and return the + * result value if one was produced. + * @param start Milliseconds (since process start) when the completion request was by the editor. + */ +export function handleGhostTextResultTelemetry<T>( + accessor: ServicesAccessor, + result: GhostTextResultWithTelemetry<T> +): T | undefined { + const logTarget = accessor.get(ICompletionsLogTargetService); + // testing/debugging only case, no telemetry + if (result.type === 'promptOnly') { return; } + + if (result.type === 'success') { + const timeToProduceMs = now() - result.telemetryBlob.issuedTime; + const reason = resultTypeToString(result.resultType); + const performanceMetrics = JSON.stringify(result.performanceMetrics); + const properties = { ...result.telemetryData, reason, performanceMetrics }; + const { foundOffset } = result.telemetryBlob.measurements; + const perf = result.performanceMetrics?.map(([key, dur]) => `\n${dur.toFixed(2)}\t${key}`).join('') ?? ''; + logger.debug( + logTarget, + `ghostText produced from ${reason} in ${Math.round(timeToProduceMs)}ms with foundOffset ${foundOffset}${perf}` + ); + telemetryRaw(accessor, 'ghostText.produced', properties, { timeToProduceMs, foundOffset }); + return result.value; + } + + logger.debug(logTarget, 'No ghostText produced -- ' + result.type + ': ' + result.reason); + if (result.type === 'canceled') { + // For backwards compatibility, we send a "fat" telemetry message in this case. + telemetry( + accessor, + `ghostText.canceled`, + result.telemetryData.telemetryBlob.extendedBy({ + reason: result.reason, + cancelledNetworkRequest: result.telemetryData.cancelledNetworkRequest ? 'true' : 'false', + }) + ); + return; + } + telemetryRaw(accessor, `ghostText.${result.type}`, { ...result.telemetryData, reason: result.reason }, {}); +} + +export function resultTypeToString(resultType: ResultType): string { + switch (resultType) { + case ResultType.Network: + return 'network'; + case ResultType.Cache: + return 'cache'; + case ResultType.Cycling: + return 'cycling'; + case ResultType.TypingAsSuggested: + return 'typingAsSuggested'; + case ResultType.Async: + return 'async'; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/asyncCompletions.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/asyncCompletions.test.ts new file mode 100644 index 0000000..8e86dfa --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/asyncCompletions.test.ts @@ -0,0 +1,310 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'node:assert'; +import sinon from 'sinon'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CancellationTokenSource } from '../../../../types/src'; +import { ICompletionsFeaturesService } from '../../experiments/featuresService'; +import { fakeAPIChoice } from '../../openai/fetch.fake'; +import { APIChoice } from '../../openai/openai'; +import { Prompt } from '../../prompt/prompt'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { delay } from '../../util/async'; +import { ResultType } from '../ghostText'; +import { AsyncCompletionManager } from './../asyncCompletions'; +import { GhostTextResultWithTelemetry, mkBasicResultTelemetry } from './../telemetry'; + +suite('AsyncCompletionManager', function () { + let accessor: ServicesAccessor; + let manager: AsyncCompletionManager; + let clock: sinon.SinonFakeTimers; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + manager = accessor.get(IInstantiationService).createInstance(AsyncCompletionManager); + clock = sinon.useFakeTimers(); + }); + + teardown(function () { + clock.restore(); + }); + + suite('shouldWaitForAsyncCompletions', function () { + test('is false when there are no requests', function () { + const prefix = 'func main() {\n'; + const prompt = createPrompt(prefix, '}\n'); + const shouldQueue = manager.shouldWaitForAsyncCompletions(prefix, prompt); + assert.strictEqual(shouldQueue, false); + }); + + test('is false when there are no matching requests', async function () { + void manager.queueCompletionRequest('0', 'import (', createPrompt(), CTS(), pendingResult()); // Prefix doesn't match + void manager.queueCompletionRequest('1', 'func main() {\n', createPrompt('', '\t'), CTS(), pendingResult()); // Suffix doesn't match + await manager.queueCompletionRequest('2', 'package ', createPrompt(), CTS(), fakeResult('main')); // Prefix doesn't match completed + await manager.queueCompletionRequest('3', 'func ', createPrompt(), CTS(), fakeResult('test')); // Completion doesn't match prefix + void manager.queueCompletionRequest('4', 'func ', createPrompt(), CTS(), pendingResult()); // Partial completion doesn't match prefix + manager.updateCompletion('4', 'func test'); + + assert.strictEqual(manager.shouldWaitForAsyncCompletions('func main() {\n', createPrompt()), false); + }); + + test('is true when there is a matching pending request', function () { + const prefix = 'func main() {\n'; + const prompt = createPrompt(prefix, '}\n'); + void manager.queueCompletionRequest('0', prefix, prompt, CTS(), pendingResult()); + + assert.strictEqual(manager.shouldWaitForAsyncCompletions(prefix, prompt), true); + }); + + test('is true when there is a matching completed request', async function () { + const prefix = 'func main() {\n'; + const prompt = createPrompt(prefix, '}\n'); + const promise = fakeResult('\tfmt.Println("Hello, world!")'); + await manager.queueCompletionRequest('0', prefix, prompt, CTS(), promise); + + assert.strictEqual(manager.shouldWaitForAsyncCompletions(prefix, prompt), true); + }); + + test('is true when there is a completed request with a prefixing prompt and matching completion', async function () { + const earlierPrefix = 'func main() {\n'; + const earlierPrompt = createPrompt(earlierPrefix, '}\n'); + const promise = fakeResult('\tfmt.Println("Hello, world!")'); + await manager.queueCompletionRequest('0', earlierPrefix, earlierPrompt, CTS(), promise); + + const prefix = 'func main() {\n\tfmt.'; + const prompt = createPrompt(prefix, '}\n'); + assert.strictEqual(manager.shouldWaitForAsyncCompletions(prefix, prompt), true); + }); + + test('is true when there is a pending request with a prefixing prompt and matching partial result', function () { + const earlierPrefix = 'func main() {\n'; + const earlierPrompt = createPrompt(earlierPrefix, '}\n'); + void manager.queueCompletionRequest('0', earlierPrefix, earlierPrompt, CTS(), pendingResult()); + manager.updateCompletion('0', '\tfmt.Println'); + + const prefix = 'func main() {\n\tfmt.'; + const prompt = createPrompt(prefix, '}\n'); + assert.strictEqual(manager.shouldWaitForAsyncCompletions(prefix, prompt), true); + }); + }); + + suite('getFirstMatchingRequest', function () { + test('returns undefined when there are no matching choices', async function () { + void manager.queueCompletionRequest('0', 'import (', createPrompt(), CTS(), pendingResult()); // Prefix doesn't match + void manager.queueCompletionRequest('1', 'func main() {\n', createPrompt('', '\t'), CTS(), pendingResult()); // Suffix doesn't match + void manager.queueCompletionRequest('2', 'func ', createPrompt(), CTS(), fakeResult('test')); // Completion doesn't match prefix + + const choice = await manager.getFirstMatchingRequest('3', 'func main() {\n', createPrompt(), false); + + assert.strictEqual(choice, undefined); + }); + + test('does not return an empty choice', async function () { + void manager.queueCompletionRequest('0', 'func ', createPrompt(), CTS(), fakeResult('main() {\n')); + + const choice = await manager.getFirstMatchingRequest('1', 'func mai(){ \n', createPrompt(), false); + + assert.strictEqual(choice, undefined); + }); + + test('returns the first resolved choice that matches', async function () { + void manager.queueCompletionRequest( + '0', + 'func ', + createPrompt(), + CTS(), + fakeResult('main() {\n', r => delay(1, r)) + ); + void manager.queueCompletionRequest( + '1', + 'func ', + createPrompt(), + CTS(), + fakeResult('main() {\n\terr :=', r => delay(2000, r)) + ); + void manager.queueCompletionRequest( + '2', + 'func ', + createPrompt(), + CTS(), + fakeResult('main() {\n\tfmt.Println', r => delay(20, r)) + ); + + const choicePromise = manager.getFirstMatchingRequest('3', 'func main() {\n', createPrompt(), false); + await clock.runAllAsync(); + const choice = await choicePromise; + + assert.ok(choice); + assert.strictEqual(choice[0].completionText, '\tfmt.Println'); + assert.strictEqual(choice[0].telemetryData.measurements.foundOffset, 9); + }); + }); + + suite('getFirstMatchingRequestWithTimeout', function () { + test('returns result before timeout', async function () { + void manager.queueCompletionRequest( + '0', + 'fmt.', + createPrompt(), + CTS(), + fakeResult('Println("Hi")', r => delay(1, r)) + ); + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.asyncCompletionsTimeout = () => 1000; + + const choicePromise = manager.getFirstMatchingRequestWithTimeout( + '1', + 'fmt.', + createPrompt(), + false, + TelemetryWithExp.createEmptyConfigForTesting() + ); + await clock.runAllAsync(); + const choice = await choicePromise; + + assert.ok(choice); + assert.strictEqual(choice[0].completionText, 'Println("Hi")'); + }); + + test('returns undefined after timeout', async function () { + void manager.queueCompletionRequest( + '0', + 'fmt.', + createPrompt(), + CTS(), + fakeResult('Println("Hello")', r => delay(2000, r)) + ); + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.asyncCompletionsTimeout = () => 10; + + const choicePromise = manager.getFirstMatchingRequestWithTimeout( + '1', + 'fmt.', + createPrompt(), + false, + TelemetryWithExp.createEmptyConfigForTesting() + ); + await clock.runAllAsync(); + const choice = await choicePromise; + + assert.strictEqual(choice, undefined); + }); + + test('does not timeout if timeout is set to -1', async function () { + void manager.queueCompletionRequest( + '0', + 'fmt.', + createPrompt(), + CTS(), + fakeResult('Println("Hi")', r => delay(100, r)) + ); + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.asyncCompletionsTimeout = () => -1; + + const choicePromise = manager.getFirstMatchingRequestWithTimeout( + '1', + 'fmt.', + createPrompt(), + false, + TelemetryWithExp.createEmptyConfigForTesting() + ); + await clock.runAllAsync(); + const choice = await choicePromise; + + assert.ok(choice); + assert.strictEqual(choice[0].completionText, 'Println("Hi")'); + }); + }); + + suite('cancels', function () { + test('pending requests that are no longer candidates for the most recent', function () { + const firstToken = CTS(); + const secondToken = CTS(); + void manager.queueCompletionRequest('0', 'import (', createPrompt(), firstToken, pendingResult()); // Prefix doesn't match + void manager.queueCompletionRequest('1', 'func ', createPrompt(), secondToken, pendingResult()); + manager.updateCompletion('1', 'test()'); // Partial completion doesn't match prefix + + void manager.getFirstMatchingRequest('2', 'func main() {\n', createPrompt(), false); + + assert.strictEqual(firstToken.token.isCancellationRequested, true); + assert.strictEqual(secondToken.token.isCancellationRequested, true); + }); + + test('pending request after updating to no longer match', function () { + const cts = CTS(); + void manager.queueCompletionRequest('1', 'func ', createPrompt(), cts, pendingResult()); + + void manager.getFirstMatchingRequest('2', 'func main() {\n', createPrompt(), false); + manager.updateCompletion('1', 'test()'); + + assert.strictEqual(cts.token.isCancellationRequested, true); + }); + + test('only requests that do not match the most recent request', function () { + const cts = CTS(); + void manager.queueCompletionRequest('1', 'func ', createPrompt(), cts, pendingResult()); + void manager.getFirstMatchingRequest('2', 'func main', createPrompt(), false); + void manager.getFirstMatchingRequest('3', 'func test', createPrompt(), false); + manager.updateCompletion('1', 'test()'); + + assert.strictEqual(cts.token.isCancellationRequested, false); + }); + + test('only requests that do not match the most recent request excluding speculative requests', function () { + const cts = CTS(); + void manager.queueCompletionRequest('1', 'func ', createPrompt(), cts, pendingResult()); + void manager.getFirstMatchingRequest('2', 'func main', createPrompt(), false); + void manager.getFirstMatchingRequest('3', 'func test', createPrompt(), false); + void manager.getFirstMatchingRequest('4', 'func main() {\nvar i;', createPrompt(), true); + manager.updateCompletion('1', 'test()'); + + assert.strictEqual(cts.token.isCancellationRequested, false); + }); + + test('all requests that do not match the most recent request', function () { + const firstCTS = CTS(); + const secondCTS = CTS(); + const thirdCTS = CTS(); + void manager.queueCompletionRequest('0', 'func ', createPrompt(), firstCTS, pendingResult()); + void manager.queueCompletionRequest('1', 'func mai', createPrompt(), secondCTS, pendingResult()); + void manager.getFirstMatchingRequest('2', 'func main', createPrompt(), false); + manager.updateCompletion('0', 'main'); + void manager.queueCompletionRequest('3', 'func t', createPrompt(), thirdCTS, pendingResult()); + void manager.getFirstMatchingRequest('4', 'func test', createPrompt(), false); + manager.updateCompletion('3', 'rigger'); + + assert.strictEqual(firstCTS.token.isCancellationRequested, true); + assert.strictEqual(secondCTS.token.isCancellationRequested, true); + assert.strictEqual(thirdCTS.token.isCancellationRequested, true); + }); + }); +}); + +function createPrompt(prefix = '', suffix = ''): Prompt { + return { prefix, suffix, isFimEnabled: true }; +} + +type Result = GhostTextResultWithTelemetry<[APIChoice, Promise<void>]>; + +function fakeResult(completionText: string, resolver = (r: Result) => Promise.resolve(r)): Promise<Result> { + const telemetryBlob = TelemetryWithExp.createEmptyConfigForTesting(); + return resolver({ + type: 'success', + value: [fakeAPIChoice(generateUuid(), 0, completionText), new Promise(() => { })], + telemetryData: mkBasicResultTelemetry(telemetryBlob), + telemetryBlob, + resultType: ResultType.Async, + }); +} + +function pendingResult(): Promise<Result> { + return new Promise(() => { }); +} + +function CTS() { + return new CancellationTokenSource(); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/blockTrimmer.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/blockTrimmer.test.ts new file mode 100644 index 0000000..dac3653 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/blockTrimmer.test.ts @@ -0,0 +1,732 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import assert from 'assert'; +import dedent from 'ts-dedent'; +import { createTextDocument } from '../../test/textDocument'; +import { TextDocumentManager } from '../../textDocumentManager'; +import { + BlockPositionType, + BlockTrimmer, + getBlockPositionType, + TerseBlockTrimmer, + VerboseBlockTrimmer, +} from '../blockTrimmer'; +const x = TextDocumentManager; +console.log(x); + +suite('VerboseBlockTrimmer', function () { + test('.getCompletionTrimOffset() returns undefined when it is under the line limit', async function () { + await testCompletionTrimming(dedent` + function twoWayMerge<T>(sortedList1: T[], sortedList2: T[]): T[] { + let mergedList: T[] = []; + let i = 0; + let j = 0; + รขยลกwhile (i < sortedList1.length && j < sortedList2.length) { + if (compareNumbers(sortedList1[i], sortedList2[j]) <= 0) { + mergedList.push(sortedList1[i]); + i++; + } else { + mergedList.push(sortedList2[j]); + j++; + } + } + `); + }); + + test('.getCompletionTrimOffset() does not trim trailing newlines', async function () { + await testCompletionTrimming( + dedent` + function twoWayMerge<T>(sortedList1: T[], sortedList2: T[]): T[] { + let mergedList: T[] = []; + let i = 0; + let j = 0; + รขยลกwhile (i < sortedList1.length && j < sortedList2.length) { + if (compareNumbers(sortedList1[i], sortedList2[j]) <= 0) { + mergedList.push(sortedList1[i]); + i++; + } else { + mergedList.push(sortedList2[j]); + j++; + } + } + ` + '\n' + ); + }); + + test('.getCompletionTrimOffset() trims to the containing block even if under the limit', async function () { + await testCompletionTrimming(dedent` + function twoWayMerge<T>(sortedList1: T[], sortedList2: T[]): T[] { + รขยลกlet mergedList: T[] = []; + let i = 0; + let j = 0; + }รขล“โ€šรฏยธย + const merged = twoWayMerge([1, 2, 3], [4, 5, 6]); + `); + }); + + test('.getCompletionTrimOffset() trims at a blank line when one is found', async function () { + await testCompletionTrimming(dedent` + function twoWayMerge<T>(sortedList1: T[], sortedList2: T[]): T[] { + รขยลกlet mergedList: T[] = []; + let i = 0; + let j = 0;รขล“โ€šรฏยธย + + while (i < sortedList1.length && j < sortedList2.length) { + if (compareNumbers(sortedList1[i], sortedList2[j]) <= 0) { + mergedList.push(sortedList1[i]); + i++; + } else { + mergedList.push(sortedList2[j]); + j++; + } + } + + while (i < sortedList1.length) { + mergedList.push(sortedList1[i]); + i++; + } + `); + }); + + test('.getCompletionTrimOffset() trims at a statement when no blank lines are present', async function () { + await testCompletionTrimming(dedent` + function twoWayMerge<T>(sortedList1: T[], sortedList2: T[]): T[] { + รขยลกlet mergedList: T[] = []; + let i = 0; + let j = 0;รขล“โ€šรฏยธย + while (i < sortedList1.length && j < sortedList2.length) { + if (compareNumbers(sortedList1[i], sortedList2[j]) <= 0) { + mergedList.push(sortedList1[i]); + i++; + } else { + mergedList.push(sortedList2[j]); + j++; + } + } + `); + }); + + test('.getCompletionTrimOffset() trims at a child statement when the first statement is over the limit', async function () { + await testCompletionTrimming(dedent` + function twoWayMerge<T>(sortedList1: T[], sortedList2: T[]): T[] { + let mergedList: T[] = []; + let i = 0; + let j = 0; + รขยลกwhile (i < sortedList1.length && j < sortedList2.length) { + // compareNumbers has the following return values: + // -1 if sortedList1[i] < sortedList2[j] + // 0 if sortedList1[i] === sortedList2[j] + // 1 if sortedList1[i] > sortedList2[j] + if (compareNumbers(sortedList1[i], sortedList2[j]) <= 0) { + // sortedList1[i] is less than or equal to sortedList2[j] + mergedList.push(sortedList1[i]); + i++; + }รขล“โ€šรฏยธย else { + // sortedList1[i] is greater than sortedList2[j] + mergedList.push(sortedList2[j]); + j++; + } + } + `); + }); + + test('.getCompletionTrimOffset() trims at a top-level statement when no containing block is present', async function () { + await testCompletionTrimming(dedent` + const a = 1; + รขยลกconst b = 2; + const c = 3; + const d = 4; + const e = 5; + const f = 6; + const g = 7; + const h = 8; + const i = 9; + const j = 10; + const k = 11;รขล“โ€šรฏยธย + const l = 12; + `); + }); + + test('.getCompletionTrimOffset() trims before a statement that begins past the line limit', async function () { + await testCompletionTrimming(dedent` + const a = 1; + รขยลกconst b = 2; + const c = 3; + const d = 4; + const e = 5; + const f = 6; + const g = 7;รขล“โ€šรฏยธย + // comment 1 + // comment 2 + // comment 3 + // comment 4 + // comment 5 + const h = 8; + `); + }); + + test('.getCompletionTrimOffset() trims trailing, non-statement text that goes over the line limit', async function () { + await testCompletionTrimming(dedent` + const a = 1; + รขยลกconst b = 2; + const c = 3; + const d = 4;รขล“โ€šรฏยธย + // comment 1 + // comment 2 + // comment 3 + // comment 4 + // comment 5 + // comment 6 + // comment 7 + // comment 8 + `); + }); + + test('.getCompletionTrimOffset() trims to the first statement when it is unsplittable even if over the limit', async function () { + await testCompletionTrimming(dedent` + function foo(arg: boolean) { + รขยลกif (arg) { + // comment 1 + // comment 2 + // comment 3 + // comment 4 + // comment 5 + // comment 6 + // comment 7 + // comment 8 + // comment 9 + // comment 10 + }รขล“โ€šรฏยธย + return; + `); + }); + + test('.getCompletionTrimOffset() trims to the first statement when it begins past the limit', async function () { + await testCompletionTrimming(dedent` + function foo(arg: boolean) { + รขยลก// comment 1 + // comment 2 + // comment 3 + // comment 4 + // comment 5 + // comment 6 + // comment 7 + // comment 8 + // comment 9 + // comment 10 + // comment 11 + const str = arg ? 'true' : 'false';รขล“โ€šรฏยธย + console.log(str); + `); + }); + + test('.getCompletionTrimOffset() trims to the first statement when it begins past the limit even if unsplittable', async function () { + await testCompletionTrimming(dedent` + function foo(arg: boolean) { + รขยลก// comment 1 + // comment 2 + // comment 3 + // comment 4 + // comment 5 + // comment 6 + // comment 7 + // comment 8 + // comment 9 + // comment 10 + // comment 11 + if (arg) { + // comment 12 + }รขล“โ€šรฏยธย + return; + `); + }); + + test('.getCompletionTrimOffset() trims to the first statement before non-statement content', async function () { + await testCompletionTrimming(dedent` + function exรขยลกample(flag) { + flag = !flag;รขล“โ€šรฏยธย + if (flag) { + flag = !flag; + if (!flag) { + flag = !flag; + if (flag) { + flag = !flag; + if (!flag) { + flag = !flag; + if (flag) { + flag = !flag; + `); + }); + + async function testCompletionTrimming(textWithCompletion: string): Promise<void> { + await testCompletionTrimmingWithTrimmer(textWithCompletion, VerboseBlockTrimmer); + } +}); + +suite('TerseBlockTrimmer', function () { + test('.getCompletionTrimOffset() returns undefined for a single statement under the line limit', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลกlet result = []; + `); + }); + + test('.getCompletionTrimOffset() trims to the containing block', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลกreturn; + }รขล“โ€šรฏยธย + function example2() { + return; + } + `); + }); + + test('.getCompletionTrimOffset() trims at a blank line', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลกlet result = [];รขล“โ€šรฏยธย + + let i = 0; + `); + }); + + test('.getCompletionTrimOffset() trims at non-statement content between statements', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลกlet result = [];รขล“โ€šรฏยธย + // comment + let i = 0; + `); + }); + + test('.getCompletionTrimOffset() trims at the start of a new compound statement', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลกlet result = []; + let i = 0;รขล“โ€šรฏยธย + for (i = 0; i < 10; i++) { + `); + }); + + test('.getCompletionTrimOffset() trims after a single compound statement', async function () { + await testCompletionTrimming(dedent` + function reverseFind(haystack, needle) { + รขยลกfor (let i = haystack.length - 1; i >= 0; i--) { + if (haystack[i] === needle) return i; + }รขล“โ€šรฏยธย + return -1; + `); + }); + + test('.getCompletionTrimOffset() trims to the line limit once the look-ahead size is exceeded', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลก// line 1 + // line 2 + // line 3รขล“โ€šรฏยธย + // line 4 + // line 5 + // line 6 + // line 7 + // line 8 + // line 9 + // line 10 + // line 11 + `); + }); + + test('.getCompletionTrimOffset() allows a single section to fill up to the look-ahead size if it is complete', async function () { + await testCompletionTrimming(dedent` + function example() { + รขยลกconst a = 1; + const b = 2; + const c = 3; + const d = 4; + const e = 5; + const f = 6;รขล“โ€šรฏยธย + while (true) { + `); + }); + + test('.getCompletionTrimOffset() supports Python', async function () { + await testCompletionTrimming( + dedent` + def reverse_find(haystack, needle): + รขยลกresult = [] + i = 0รขล“โ€šรฏยธย + while i < len(haystack): + `, + 'python' + ); + }); + + test('.getCompletionTrimOffset() trims to a containing block in Python', async function () { + await testCompletionTrimming( + dedent` + def example(a, b): + if a > b: + รขยลกc = a - b + return cรขล“โ€šรฏยธย + else: + `, + 'python' + ); + }); + + test('.getCompletionTrimOffset() supports Go', async function () { + await testCompletionTrimming( + dedent` + package main + + func reverseFind(haystack []int, needle int) int { + รขยลกresult := []int{} + i := 0รขล“โ€šรฏยธย + for i < len(haystack) { + if haystack[i] == needle { + return i + `, + 'go' + ); + }); + + test('.getCompletionTrimOffset() supports PHP', async function () { + await testCompletionTrimming( + dedent` + <?php + function reverse_find($haystack, $needle) { + รขยลก$search = array_reverse($haystack, true);รขล“โ€šรฏยธย + foreach ($search as $index => $item) { + if ($item === $needle) { + return $index; + } + } + `, + 'php' + ); + }); + + test('.getCompletionTrimOffset() supports Ruby', async function () { + await testCompletionTrimming( + dedent` + def reverse_find(haystack, needle) + รขยลกlen = haystack.length + i = len - 1รขล“โ€šรฏยธย + while i >= 0 do + return i if haystack[i] == needle + i -= 1 + end + `, + 'ruby' + ); + }); + + test('.getCompletionTrimOffset() supports Java', async function () { + await testCompletionTrimming( + dedent` + public class Main { + public static int reverseFind(int[] haystack, int needle) { + รขยลกint end = haystack.length - 1;รขล“โ€šรฏยธย + for (int i = end; i >= 0; i--) { + if (haystack[i] == needle) { + return i; + } + } + `, + 'java' + ); + }); + + test('.getCompletionTrimOffset() supports C#', async function () { + await testCompletionTrimming( + dedent` + class Program { + static int ReverseFind(int[] haystack, int needle) { + รขยลกint end = haystack.Length - 1;รขล“โ€šรฏยธย + for (int i = end; i >= 0; i--) { + if (haystack[i] == needle) { + return i; + } + } + `, + 'csharp' + ); + }); + + test('.getCompletionTrimOffset() supports C', async function () { + await testCompletionTrimming( + dedent` + #include <stdio.h> + + int reverse_find(int haystack[], int needle, int size) { + รขยลกint i = size - 1;รขล“โ€šรฏยธย + while (i >= 0) { + if (haystack[i] == needle) { + return i; + } + i--; + } + `, + 'c' + ); + }); + + test('.getCompletionTrimOffset() supports C++', async function () { + await testCompletionTrimming( + dedent` + #include <iostream> + using namespace std; + + template <typename T> + class ReverseFind { + public: + static int find(T haystack[], T needle, int size) { + รขยลกint i = size - 1;รขล“โ€šรฏยธย + while (i >= 0) { + if (haystack[i] == needle) { + return i; + } + i--; + } + } + }; + `, + 'cpp' + ); + }); + + async function testCompletionTrimming(textWithCompletion: string, languageId = 'typescript'): Promise<void> { + await testCompletionTrimmingWithTrimmer(textWithCompletion, TerseBlockTrimmer, languageId); + } +}); + +interface BlockTrimmerConstructor { + new(languageId: string, prefix: string, completion: string): BlockTrimmer; +} + +async function testCompletionTrimmingWithTrimmer( + textWithCompletion: string, + blockTrimmerType: BlockTrimmerConstructor, + languageId = 'typescript' +): Promise<void> { + const cursorMarker = 'รขยลก'; + const trimMarker = 'รขล“โ€šรฏยธย'; + const cursorPos = textWithCompletion.indexOf(cursorMarker); + const trimPos = textWithCompletion.indexOf(trimMarker); + const prefix = textWithCompletion.substring(0, cursorPos); + const trimmed = textWithCompletion.substring(cursorPos + cursorMarker.length, trimPos === -1 ? undefined : trimPos); + const completion = trimmed + (trimPos === -1 ? '' : textWithCompletion.substring(trimPos + trimMarker.length)); + const expectedOffset = trimPos === -1 ? undefined : trimmed.length; + const trimmer = new blockTrimmerType(languageId, prefix, completion); + + const actualOffset = await trimmer.getCompletionTrimOffset(); + + assert.strictEqual( + actualOffset, + expectedOffset, + dedent` + Expected an offset of ${expectedOffset} but got ${actualOffset} + ${trimmed === completion.substring(0, actualOffset) ? 'true' : 'false'} + + expected completion: + ${JSON.stringify(completion.substring(0, expectedOffset))} + + actual completion: + ${JSON.stringify(completion.substring(0, actualOffset))} + ` + ); +} + +suite('getBlockPositionType()', function () { + test('empty document returns NonBlock', async function () { + await testPositionType(BlockPositionType.NonBlock, 'รขยลก'); + }); + + test('on a simple expression returns NonBlock', async function () { + await testPositionType(BlockPositionType.NonBlock, 'รขยลกconst x = 1;'); + }); + + test('with an empty block returns EmptyBlock', async function () { + await testPositionType(BlockPositionType.EmptyBlock, 'while (true) { รขยลก }'); + await testPositionType(BlockPositionType.EmptyBlock, 'function example() { รขยลก }'); + }); + + test('at the end of a non-empty block returns BlockEnd', async function () { + await testPositionType(BlockPositionType.BlockEnd, 'while (true) { x += 1; รขยลก }'); + }); + + test('mid-statement at the end of a non-empty block returns BlockEnd', async function () { + await testPositionType(BlockPositionType.BlockEnd, 'while (true) { x += 1รขยลก; }'); + }); + + test('between statements within a block returns MidBlock', async function () { + await testPositionType(BlockPositionType.MidBlock, 'while (true) { last = x; รขยลก x += 1; }'); + }); + + test('on a statement before the last within a block returns MidBlock', async function () { + await testPositionType(BlockPositionType.MidBlock, 'while (true) { last = xรขยลก; x += 1; }'); + }); + + test('on a multi-line simple statement within a block before the last line returns MidBlock', async function () { + await testPositionType( + BlockPositionType.MidBlock, + dedent` + if (true) { + someFunction( + arg1, + arg2, + รขยลก + arg3 + ); + } + ` + ); + }); + + test('on a multi-line simple statement within a block on the last line returns BlockEnd', async function () { + await testPositionType( + BlockPositionType.BlockEnd, + dedent` + if (true) { + someFunction( + arg1, + arg2, + arg3 + รขยลก); + } + ` + ); + }); + + // confirm single-line if statement behavior in JS given the special treatment by StatementTree: + test('inside an empty block of a single-line if statement in JS returns EmptyBlock', async function () { + await testPositionType(BlockPositionType.EmptyBlock, 'if (true) { รขยลก }'); + }); + + test('supports Python', async function () { + await testPositionType( + BlockPositionType.MidBlock, + dedent` + def example(): + รขยลก + pass + `, + 'python' + ); + }); + + test('supports Go', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + package main + + func main() { + รขยลก + } + `, + 'go' + ); + }); + + test('supports PHP', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + <?php + function main() { + รขยลก + } + `, + 'php' + ); + }); + + test('supports Ruby', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + def main + รขยลก + end + `, + 'ruby' + ); + }); + + test('supports Java', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + public class Main { + public static void main(String[] args) { + รขยลก + } + } + `, + 'java' + ); + }); + + test('supports C#', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + class Program + { + static void Main(string[] args) { + รขยลก + } + } + `, + 'csharp' + ); + }); + + test('supports C', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + #include <iostream> + + int main() { + รขยลก + } + `, + 'cpp' + ); + }); + + test('supports C++', async function () { + await testPositionType( + BlockPositionType.EmptyBlock, + dedent` + #include <iostream> + + class Main { + รขยลก + } + `, + 'cpp' + ); + }); + + async function testPositionType( + expectedType: BlockPositionType, + textWithCursor: string, + languageId = 'typescript' + ): Promise<void> { + const cursorMarker = 'รขยลก'; + const cursorPos = textWithCursor.indexOf(cursorMarker); + const prefix = textWithCursor.substring(0, cursorPos); + const suffix = textWithCursor.substring(cursorPos + cursorMarker.length); + const doc = createTextDocument('file:///test.ts', languageId, 0, prefix + suffix); + const pos = doc.positionAt(cursorPos); + + const actualType = await getBlockPositionType(doc, pos); + + assert.strictEqual(actualType, expectedType); + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/current.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/current.test.ts new file mode 100644 index 0000000..0db9bce --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/current.test.ts @@ -0,0 +1,152 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CurrentGhostText } from '../current'; +import { ResultType } from '../ghostText'; +import { fakeAPIChoice } from '../../openai/fetch.fake'; +import { APIChoice } from '../../openai/openai'; +import * as assert from 'assert'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; + +suite('CurrentGhostText', function () { + let current: CurrentGhostText; + + setup(function () { + current = new CurrentGhostText(); + }); + + suite('getCompletionsForUserTyping', function () { + test('returns undefined if there is no current completion', function () { + const result = current.getCompletionsForUserTyping('func main() {\n', ''); + assert.strictEqual(result, undefined); + }); + + test('returns the current completion for an exact match', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func main() {\n', ''); + + assert.deepStrictEqual(result, [choice]); + }); + + test('returns the current completion for a prefix match', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func main() {\nfmt.Print', ''); + + assert.deepStrictEqual(result, [{ ...choice, completionText: 'ln("Hello, World!")' }]); + }); + + test('returns undefined when the prefix does not match', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func test() {\n', '}'); + + assert.strictEqual(result, undefined); + }); + + test('returns undefined when the suffix does not match', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func main() {\n', '}'); + assert.strictEqual(result, undefined); + }); + + test('returns undefined when the completion does not match', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func main() {\nerr', '}'); + assert.strictEqual(result, undefined); + }); + + test('returns undefined when the completion is exhausted', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func main() {\nfmt.Println("Hello, World!")', ''); + assert.strictEqual(result, undefined); + }); + + test('does not change the current completion when TypingAsSuggested', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + current.setGhostText( + 'func main() {\nfmt.', + '', + [fakeChoice('Println("Hello, World!")')], + ResultType.TypingAsSuggested + ); + const result = current.getCompletionsForUserTyping('func main() {\n', ''); + + assert.deepStrictEqual(result![0].requestId, choice.requestId); + }); + + test('only returns cycling completions that match', function () { + const choice = fakeChoice(); + const choice2 = fakeChoice('err := nil', 1); + const choice3 = fakeChoice('fmt.Println("hi")', 2); + current.setGhostText('func main() {\n', '', [choice, choice2, choice3], ResultType.Network); + + const result = current.getCompletionsForUserTyping('func main() {\nfmt', ''); + + assert.deepStrictEqual(result, [ + { ...choice, completionText: '.Println("Hello, World!")' }, + { ...choice3, completionText: '.Println("hi")' }, + ]); + }); + }); + + suite('hasAcceptedCurrentCompletion', function () { + test('returns false if there is no current completion', function () { + assert.ok(!current.hasAcceptedCurrentCompletion('func main() {\n', '')); + }); + + test('returns false for uncompleted completions', function () { + current.setGhostText('func main() {\n', '', [fakeChoice()], ResultType.Network); + + assert.ok(!current.hasAcceptedCurrentCompletion('func main() {\n', '')); + assert.ok(!current.hasAcceptedCurrentCompletion('func main() {\nfmt.Println', '')); + assert.ok(!current.hasAcceptedCurrentCompletion('func main() {\nfmt.Println("hi")', '')); + }); + + test('returns true for completed completion', function () { + current.setGhostText('func main() {\n', '', [fakeChoice()], ResultType.Network); + + assert.ok(current.hasAcceptedCurrentCompletion('func main() {\nfmt.Println("Hello, World!")', '')); + }); + + test('returns false for completed completion with content_filter finish reason', function () { + const choice = fakeChoice(); + choice.finishReason = 'content_filter'; + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + assert.ok(!current.hasAcceptedCurrentCompletion('func main() {\nfmt.Println("Hello, World!")', '')); + }); + + test('returns false for completed completion with snippy finish reason', function () { + const choice = fakeChoice(); + choice.finishReason = 'snippy'; + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + assert.ok(!current.hasAcceptedCurrentCompletion('func main() {\nfmt.Println("Hello, World!")', '')); + }); + }); + + test('clientCompletionId returns the current completion id', function () { + const choice = fakeChoice(); + current.setGhostText('func main() {\n', '', [choice], ResultType.Network); + + assert.strictEqual(current.clientCompletionId, choice.clientCompletionId); + }); +}); + +function fakeChoice(completionText = 'fmt.Println("Hello, World!")', choice = 0): APIChoice { + return fakeAPIChoice(generateUuid(), choice, completionText); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/ghostText.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/ghostText.test.ts new file mode 100644 index 0000000..71c956d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/ghostText.test.ts @@ -0,0 +1,676 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import assert from 'assert'; +import { Position } from 'shiki/core'; +import dedent from 'ts-dedent'; +import type { CancellationToken } from 'vscode'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; +import { SyncDescriptor } from '../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { initializeTokenizers } from '../../../../prompt/src/tokenization'; +import { CompletionState, createCompletionState } from '../../completionState'; +import { ConfigKey, ICompletionsConfigProvider, InMemoryConfigProvider } from '../../config'; +import { ICompletionsFetcherService, Response } from '../../networking'; +import { ICompletionsOpenAIFetcherService, LiveOpenAIFetcher } from '../../openai/fetch'; +import { fakeAPIChoice, fakeAPIChoiceFromCompletion } from '../../openai/fetch.fake'; +import { APIChoice } from '../../openai/openai'; +import { extractPrompt, PromptResponsePresent, trimLastLine } from '../../prompt/prompt'; +import { getGhostTextInternal } from '../../prompt/test/prompt'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { createFakeCompletionResponse, fakeCodeReference, NoFetchFetcher, StaticFetcher } from '../../test/fetcher'; +import { withInMemoryTelemetry } from '../../test/telemetry'; +import { createTextDocument } from '../../test/textDocument'; +import { ITextDocument, LocationFactory } from '../../textDocument'; +import { Deferred } from '../../util/async'; +import { ICompletionsAsyncManagerService } from '../asyncCompletions'; +import { ICompletionsCacheService } from '../completionsCache'; +import { ICompletionsCurrentGhostText } from '../current'; +import { getGhostText, GetNetworkCompletionsType, GhostCompletion, ResultType } from '../ghostText'; +import { mkBasicResultTelemetry } from '../telemetry'; + +// Unit tests for ghostText that do not require network connectivity. For other +// tests, see lib/e2e/src/ghostText.test.ts. + +suite('Isolated GhostText tests', function () { + function getPrefix(completionState: CompletionState): string { + return trimLastLine( + completionState.textDocument.getText( + LocationFactory.range(LocationFactory.position(0, 0), completionState.position) + ) + )[0]; + } + + function setupCompletion( + fetcher: ICompletionsFetcherService, + docText = 'import "fmt"\n\nfunc fizzbuzz(n int) {\n\n}\n', + position = LocationFactory.position(3, 0), + languageId = 'go', + token?: CancellationToken + ) { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsFetcherService, fetcher); + serviceCollection.define(ICompletionsOpenAIFetcherService, new SyncDescriptor(LiveOpenAIFetcher)); // gets results from static fetcher + const accessor = serviceCollection.createTestingAccessor(); + + const doc = createTextDocument('file:///fizzbuzz.go', languageId, 1, docText); + const state = createCompletionState(doc, position); + const prefix = getPrefix(state); + + // Setup closures with the state as default + function requestGhostText(completionState = state) { + return getGhostText(accessor, completionState, token, {}); + } + async function requestPrompt(completionState = state) { + const telemExp = TelemetryWithExp.createEmptyConfigForTesting(); + const result = await extractPrompt(accessor, 'COMPLETION_ID', completionState, telemExp); + return (result as PromptResponsePresent).prompt; + } + + // Note, that we return a copy of the state to avoid side effects + return { + accessor, + doc, + position, + prefix, + state: createCompletionState(doc, position), + requestGhostText, + requestPrompt, + }; + } + + function addToCache(accessor: ServicesAccessor, prefix: string, suffix: string, completion: string | APIChoice) { + let choice: APIChoice; + if (typeof completion === 'string') { + choice = fakeAPIChoiceFromCompletion(completion); + } else { + choice = completion; + } + const cache = accessor.get(ICompletionsCacheService); + cache.append(prefix, suffix, choice); + } + + async function acceptAndRequestNextCompletion( + accessor: ServicesAccessor, + origDoc: ITextDocument, + origPosition: Position, + completion: GhostCompletion + ) { + const doc = createTextDocument( + origDoc.uri, + origDoc.clientLanguageId, + origDoc.version + 1, + origDoc.getText(LocationFactory.range(LocationFactory.position(0, 0), origPosition)) + + completion.completionText + + origDoc.getText(LocationFactory.range(origPosition, origDoc.positionAt(origDoc.getText().length))) + ); + const position = doc.positionAt(doc.offsetAt(origPosition) + completion.completionText.length); + const result = await getGhostTextInternal(accessor, doc, position); + return { doc, position, result }; + } + + suiteSetup(async function () { + await initializeTokenizers; + }); + + test('returns annotations in the result', async function () { + const { requestGhostText } = setupCompletion( + new StaticFetcher(() => + createFakeCompletionResponse('\tfor i := 1; i<= n; i++ {\n', { + annotations: fakeCodeReference(-18, 26, 'NOASSERTION', 'https://github.com/github/example'), + }) + ) + ); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.deepStrictEqual(responseWithTelemetry.value[0][0].copilotAnnotations?.ip_code_citations, [ + { + id: 5, + start_offset: -18, + stop_offset: 26, + details: { citations: [{ url: 'https://github.com/github/example', license: 'NOASSERTION' }] }, + }, + ]); + }); + + test('returns cached completion', async function () { + const { accessor, requestGhostText, prefix, requestPrompt } = setupCompletion(new NoFetchFetcher()); + const completionText = '\tfor i := 1; i<= n; i++ {'; + const { suffix } = await requestPrompt(); + addToCache(accessor, prefix, suffix, completionText); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.strictEqual(responseWithTelemetry.value[0][0].completion.completionText, completionText); + assert.strictEqual(responseWithTelemetry.value[1], ResultType.Cache, 'result type should be cache'); + }); + + test('returns empty response when cached completion is filtered by post-processing', async function () { + const completionText = '\tvar i int'; + const { accessor, requestGhostText, prefix, requestPrompt } = setupCompletion( + new StaticFetcher(() => createFakeCompletionResponse(completionText)) + ); + const { suffix } = await requestPrompt(); + addToCache(accessor, prefix, suffix, '}'); // Completion matches next line of document + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'empty'); + assert.strictEqual(responseWithTelemetry.reason, 'cached results empty after post-processing'); + }); + + test('returns typing as suggested', async function () { + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion(new NoFetchFetcher()); + const { suffix } = await requestPrompt(); + addToCache(accessor, prefix, suffix, '\tfor i := 1; i<= n; i++ {'); + await requestGhostText(); + + const secondText = 'import "fmt"\n\nfunc fizzbuzz(n int) {\n\tfor\n}\n'; + const second = createCompletionState( + createTextDocument('file:///fizzbuzz.go', 'go', 1, secondText), + LocationFactory.position(3, 4) + ); + const responseWithTelemetry = await requestGhostText(second); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.strictEqual(responseWithTelemetry.value[0][0].completion.completionText, ' i := 1; i<= n; i++ {'); + assert.strictEqual( + responseWithTelemetry.value[1], + ResultType.TypingAsSuggested, + 'result type should be typing as suggested' + ); + }); + + test('returns multiline typing as suggested when typing into single line context', async function () { + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion(new NoFetchFetcher()); + const currentGhostText = accessor.get(ICompletionsCurrentGhostText); + currentGhostText.hasAcceptedCurrentCompletion = () => true; + const { suffix } = await requestPrompt(); + const completionText = '\tfmt.Println("hi")\n\tfmt.Print("hello")'; + addToCache(accessor, prefix, suffix, completionText); + const firstRes = await requestGhostText(); + assert.strictEqual(firstRes.type, 'success'); + assert.strictEqual(firstRes.value[0][0].completion.completionText, completionText); + + // Request a second completion typing into a non-multiline context: + // the addition of `\tfmt.` to the current line changes the completion + // context (via the `isEmptyBlockStart` computed in prompt/) from + // multiline to single line. + const secondText = 'import "fmt"\n\nfunc fizzbuzz(n int) {\n\tfmt.\n}\n'; + const second = createCompletionState( + createTextDocument('file:///fizzbuzz.go', 'go', 1, secondText), + LocationFactory.position(3, 9) + ); + const secondRes = await requestGhostText(second); + + assert.strictEqual(secondRes.type, 'success'); + assert.strictEqual(secondRes.value[0][0].completion.completionText, 'Println("hi")\n\tfmt.Print("hello")'); + assert.strictEqual(secondRes.value[1], ResultType.TypingAsSuggested); + }); + + test('trims multiline async completion into single line context', async function () { + const { accessor, doc, position, requestGhostText, requestPrompt } = setupCompletion(new NoFetchFetcher()); + const asyncManager = accessor.get(ICompletionsAsyncManagerService); + const prompt = await requestPrompt(); + const [prefix] = trimLastLine(doc.getText(LocationFactory.range(LocationFactory.position(0, 0), position))); + const response = fakeResult('\tfmt.Println("hi")\n\tfmt.Print("hello")'); + void asyncManager.queueCompletionRequest('0', prefix, prompt, new CancellationTokenSource(), response); + + // Request a single completion by typing into a non-multiline context: + // the addition of `\tfmt.` to the current line changes the completion + // context (via the `isEmptyBlockStart` computed in prompt/) from + // multiline to single line. + const secondText = 'import "fmt"\n\nfunc fizzbuzz(n int) {\n\tfmt.\n}\n'; + const second = createCompletionState( + createTextDocument('file:///fizzbuzz.go', 'go', 1, secondText), + LocationFactory.position(3, 9) + ); + const secondRes = await requestGhostText(second); + + assert.strictEqual(secondRes.type, 'success'); + assert.strictEqual(secondRes.value[0][0].completion.completionText, 'Println("hi")'); + assert.strictEqual(secondRes.value[1], ResultType.Async); + }); + + test('returns cached single-line completion that starts with newline', async function () { + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion( + new NoFetchFetcher(), + 'import "fmt"\n\nfunc fizzbuzz(n int) {\n\ti := 0\n}\n', + LocationFactory.position(3, '\ti := 0'.length) + ); + const { suffix } = await requestPrompt(); + const completionText = '\n\tj := 0'; + addToCache(accessor, prefix, suffix, completionText); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.strictEqual(responseWithTelemetry.value[0][0].completion.completionText, completionText); + assert.strictEqual(responseWithTelemetry.value[1], ResultType.Cache, 'result type should be cache'); + }); + + test('returns prefixed cached completion', async function () { + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion(new NoFetchFetcher()); + const { suffix } = await requestPrompt(); + const earlierPrefix = prefix.substring(0, prefix.length - 3); + const remainingPrefix = prefix.substring(prefix.length - 3); + const completionText = '\tfor i := 1; i<= n; i++ {'; + addToCache(accessor, earlierPrefix, suffix, remainingPrefix + completionText); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.strictEqual(responseWithTelemetry.value[0][0].completion.completionText, completionText); + assert.strictEqual(responseWithTelemetry.value[1], ResultType.Cache, 'result type should be cache'); + assert.strictEqual(responseWithTelemetry.telemetryBlob.measurements.foundOffset, 3); + }); + + test('does not return cached completion when exhausted', async function () { + const networkCompletionText = '\tfor i := 1; i<= n; i++ {'; + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion( + new StaticFetcher(() => { + return createFakeCompletionResponse(networkCompletionText); + }) + ); + const { suffix } = await requestPrompt(); + const earlierPrefix = prefix.substring(0, prefix.length - 3); + const remainingPrefix = prefix.substring(prefix.length - 3); + addToCache(accessor, earlierPrefix, suffix, remainingPrefix); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.strictEqual(responseWithTelemetry.value[0][0].completion.completionText, networkCompletionText); + assert.strictEqual(responseWithTelemetry.value[1], ResultType.Async, 'result type should be async'); + }); + + test('Multiline requests return multiple completions on second invocation', async function () { + const firstCompletionText = '\tfirstVar := 1\n'; + const secondCompletionText = '\tfirstVar := 2\t'; + const completions = [firstCompletionText, secondCompletionText]; + let serverSentResponse = false; + const { requestGhostText } = setupCompletion( + new StaticFetcher((url, options) => { + if (serverSentResponse) { + throw new Error('Unexpected second request'); + } + serverSentResponse = true; + return createFakeCompletionResponse(completions); + }) + ); + // Get the completion from the server, do the processing of the responses + // this is a multiline request, so it'll request multiple completions, but whatever our cycling specification, it'll not _wait_ for those, c.f isCyclingRequest in getGhostTextStrategy. + const firstResponse = await requestGhostText(); + assert.strictEqual(firstResponse.type, 'success'); + assert.strictEqual(firstResponse.value[0].length, 1); + assert.strictEqual(firstResponse.value[0][0].completion.completionText, firstCompletionText.trimEnd()); + // therefore, request the same prompt again, this time with cycling specified, to get all completions from the cache + const secondResponse = await requestGhostText(); + assert.strictEqual(secondResponse.type, 'success'); + // two completion results returned + assert.strictEqual(secondResponse.value[0].length, 2); + // the second one is the second completion, but with whitespace trimmed + assert.strictEqual(secondResponse.value[0][0].completion.completionText, firstCompletionText.trimEnd()); + assert.strictEqual(secondResponse.value[0][1].completion.completionText, secondCompletionText.trimEnd()); + }); + + test('Responses with duplicate content (modulo whitespace) are deduplicated', async function () { + const firstCompletionText = '\tfirstVar := 1\n'; + const secondCompletionText = '\tfirstVar := 1\t'; + const completions = [firstCompletionText, secondCompletionText]; + let serverSentResponse = false; + const { requestGhostText } = setupCompletion( + new StaticFetcher((url, options) => { + if (serverSentResponse) { + throw new Error('Unexpected second request'); + } + serverSentResponse = true; + return createFakeCompletionResponse(completions); + }) + ); + // Get the completion from the server, do the processing of the responses + // this is a multiline request, so it'll request multiple completions, but whatever our cycling specification, it'll not _wait_ for those, c.f isCyclingRequest in getGhostTextStrategy. + const firstResponse = await requestGhostText(); + assert.strictEqual(firstResponse.type, 'success'); + assert.strictEqual(firstResponse.value[0].length, 1); + assert.strictEqual(firstResponse.value[0][0].completion.completionText, firstCompletionText.trimEnd()); + // therefore, request the same prompt again, this time with cycling specified, to get all completions from the cache + const secondResponse = await requestGhostText(); + assert.strictEqual(secondResponse.type, 'success'); + // still only one completion result returned + assert.strictEqual(secondResponse.value[0].length, 1); + assert.strictEqual(secondResponse.value[0][0].completion.completionText, firstCompletionText.trimEnd()); + }); + + test('adds prompt metadata to telemetry', async function () { + const networkCompletionText = '\tfor i := 1; i<= n; i++ {'; + const { accessor, requestGhostText } = setupCompletion( + new StaticFetcher(() => { + return createFakeCompletionResponse(networkCompletionText); + }) + ); + + const { result, reporter } = await withInMemoryTelemetry(accessor, async () => { + return await requestGhostText(); + }); + + // The returned object (used for all other telemetry events) does not have the prompt metadata + assert.deepStrictEqual(result.type, 'success'); + assert.ok(!result.telemetryBlob.properties.promptMetadata); + + // Only the issued event has it + const issuedTelemetry = reporter.eventByName('ghostText.issued'); + assert.ok(issuedTelemetry.properties.promptMetadata); + + // Double check that the other events don't have it + const events = reporter.events.filter(e => e.name !== 'ghostText.issued'); + assert.ok(events.length > 0); + for (const event of events) { + assert.ok(!event.properties.promptMetadata); + } + }); + + test('cache hits use issuedTime in telemetry from current request, not cache', async function () { + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion(new NoFetchFetcher()); + const { suffix } = await requestPrompt(); + const completionText = '\tfor i := 1; i<= n; i++ {'; + const choice = fakeAPIChoiceFromCompletion(completionText); + choice.telemetryData.issuedTime -= 100; + addToCache(accessor, prefix, suffix, completionText); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual( + responseWithTelemetry.value[0][0].telemetry.issuedTime, + responseWithTelemetry.telemetryBlob.issuedTime + ); + }); + + test('sends ghostText.issued telemetry event', async function () { + const networkCompletionText = '\tfor i := 1; i<= n; i++ {'; + const { accessor, requestGhostText } = setupCompletion( + new StaticFetcher(() => { + return createFakeCompletionResponse(networkCompletionText); + }) + ); + + const { result, reporter } = await withInMemoryTelemetry(accessor, async () => { + return await requestGhostText(); + }); + + assert.strictEqual(result.type, 'success'); + const issuedTelemetry = reporter.eventByName('ghostText.issued'); + [ + 'languageId', + 'beforeCursorWhitespace', + 'afterCursorWhitespace', + 'neighborSource', + 'gitRepoInformation', + 'engineName', + 'isMultiline', + 'blockMode', + 'isCycling', + ].forEach(prop => { + assert.strictEqual( + typeof issuedTelemetry.properties[prop], + 'string', + `Expected telemetry property ${prop}` + ); + }); + [ + 'promptCharLen', + 'promptSuffixCharLen', + 'promptEndPos', + 'documentLength', + 'documentLineCount', + 'promptComputeTimeMs', + ].forEach(prop => { + assert.strictEqual( + typeof issuedTelemetry.measurements[prop], + 'number', + `Expected telemetry measurement ${prop}` + ); + }); + }); + + test('excludes ghostText.issued-specific propeties in returned telemetry', async function () { + const networkCompletionText = '\tfor i := 1; i<= n; i++ {'; + const { requestGhostText } = setupCompletion( + new StaticFetcher(() => { + return createFakeCompletionResponse(networkCompletionText); + }) + ); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + [ + 'beforeCursorWhitespace', + 'afterCursorWhitespace', + 'promptChoices', + 'promptBackground', + 'neighborSource', + 'blockMode', + ].forEach(prop => { + assert.strictEqual( + responseWithTelemetry.value[0][0].telemetry.properties[prop], + undefined, + `Did not expect telemetry property ${prop}` + ); + assert.strictEqual( + responseWithTelemetry.telemetryBlob.properties[prop], + undefined, + `Did not expect telemetry property ${prop}` + ); + }); + ['promptCharLen', 'promptSuffixCharLen', 'promptCharLen', 'promptEndPos', 'promptComputeTimeMs'].forEach( + prop => { + assert.strictEqual( + responseWithTelemetry.value[0][0].telemetry.measurements[prop], + undefined, + `Did not expect telemetry measurement ${prop}` + ); + assert.strictEqual( + responseWithTelemetry.telemetryBlob.measurements[prop], + undefined, + `Did not expect telemetry measurement ${prop}` + ); + } + ); + }); + + test('includes document information in returned telemetry', async function () { + const networkCompletionText = '\tfor i := 1; i<= n; i++ {'; + const { requestGhostText } = setupCompletion( + new StaticFetcher(() => { + return createFakeCompletionResponse(networkCompletionText); + }) + ); + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + ['languageId', 'gitRepoInformation', 'engineName', 'isMultiline', 'isCycling'].forEach(prop => { + assert.strictEqual( + typeof responseWithTelemetry.value[0][0].telemetry.properties[prop], + 'string', + `Expected telemetry property ${prop}` + ); + assert.strictEqual( + typeof responseWithTelemetry.telemetryBlob.properties[prop], + 'string', + `Expected telemetry property ${prop}` + ); + }); + }); + + test('updates transient document information in telemetry of cached choices', async function () { + const { accessor, requestGhostText, requestPrompt, prefix } = setupCompletion(new NoFetchFetcher()); + const { suffix } = await requestPrompt(); + const completionText = '\tfor i := 1; i<= n; i++ {'; + addToCache(accessor, prefix, suffix, completionText); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + ['documentLength', 'documentLineCount'].forEach(prop => { + assert.strictEqual( + typeof responseWithTelemetry.telemetryBlob.measurements[prop], + 'number', + `Expected telemetry measurement ${prop}` + ); + assert.strictEqual( + responseWithTelemetry.value[0][0].telemetry.measurements[prop], + responseWithTelemetry.telemetryBlob.measurements[prop], + `Expected telemetry measurement ${prop} to be ${responseWithTelemetry.telemetryBlob.measurements[prop]}` + ); + }); + }); + + test('cancels if token is canceled', async function () { + const tokenSource = new CancellationTokenSource(); + const deferredResponse = new Deferred<Response>(); + const { requestGhostText } = setupCompletion( + new StaticFetcher(() => deferredResponse.promise), + undefined, + undefined, + undefined, + tokenSource.token + ); + + const requestPromise = requestGhostText(); + tokenSource.cancel(); + deferredResponse.resolve(createFakeCompletionResponse('var i int')); + const result = await requestPromise; + + assert.strictEqual(result.type, 'abortedBeforeIssued'); + assert.strictEqual(result.reason, 'cancelled before extractPrompt'); + }); + + test('cancels if a newer completion request is made', async function () { + const firstResponseDeferred = new Deferred<Response>(); + const secondResponseDeferred = new Deferred<Response>(); + const deferreds = [firstResponseDeferred, secondResponseDeferred]; + const { requestGhostText } = setupCompletion(new StaticFetcher(() => deferreds.shift()!.promise)); + + const firstResponsePromise = requestGhostText(); + const secondResponsePromise = requestGhostText(); + firstResponseDeferred.resolve(createFakeCompletionResponse('var i int')); + secondResponseDeferred.resolve(createFakeCompletionResponse('var j int')); + const firstResponse = await firstResponsePromise; + const secondResponse = await secondResponsePromise; + + assert.strictEqual(firstResponse.type, 'abortedBeforeIssued'); + assert.strictEqual(firstResponse.reason, 'cancelled before extractPrompt'); + assert.strictEqual(secondResponse.type, 'success'); + }); + + test('can close an unclosed brace (when using progressive reveal)', async function () { + const { accessor, requestGhostText } = setupCompletion( + new StaticFetcher(() => createFakeCompletionResponse(' }\n')), + dedent` + function hello(n: number) { + for (let i = 1; i<= n; i++) { + console.log("hello") + + } + `, + LocationFactory.position(3, 0), + 'typescript' + ); + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.AlwaysRequestMultiline, true); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 1); + assert.strictEqual(responseWithTelemetry.value[0][0].completion.completionText, ' }'); + }); + + test('filters out a duplicate brace (when using progressive reveal)', async function () { + const { accessor, requestGhostText } = setupCompletion( + new StaticFetcher(() => createFakeCompletionResponse('}\n')), + dedent` + function hello(n: number) { + for (let i = 1; i<= n; i++) { + console.log("hello") + } + + } + `, + LocationFactory.position(4, 0), + 'typescript' + ); + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.AlwaysRequestMultiline, true); + + const responseWithTelemetry = await requestGhostText(); + + assert.strictEqual(responseWithTelemetry.type, 'success'); + assert.strictEqual(responseWithTelemetry.value[0].length, 0); + }); + + test('progressive reveal uses a speculative request for multiline completions and caches further completions', async function () { + const raw = dedent` + switch { + case n%3 == 0: + output += "Fizz" + fallthrough + case n%5 == 0: + output += "Buzz" + default: + output = fmt.Sprintf("%d", n) + } + fmt.Println(output) + `; + const lines = raw.split('\n').map(line => ` ${line}`); + const multilineCompletion = lines.join('\n'); + const { accessor, doc, position, state } = setupCompletion( + new StaticFetcher(() => createFakeCompletionResponse(multilineCompletion)) + ); + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + const currentGhostText = accessor.get(ICompletionsCurrentGhostText); + configProvider.setConfig(ConfigKey.AlwaysRequestMultiline, true); + currentGhostText.hasAcceptedCurrentCompletion = () => true; + + const response = await getGhostText(accessor, state, undefined, { isSpeculative: true }); + + assert.strictEqual(response.type, 'success'); + assert.strictEqual(response.value[0].length, 1); + assert.strictEqual(response.value[0][0].completion.completionText, lines.slice(0, 9).join('\n')); + + const { result } = await acceptAndRequestNextCompletion(accessor, doc, position, response.value[0][0].completion); + + assert.strictEqual(result.type, 'success'); + assert.strictEqual(result.value[0].length, 1); + assert.strictEqual(result.value[0][0].completion.completionText, '\n' + lines.slice(9).join('\n')); + assert.strictEqual(result.resultType, ResultType.Cache); + }); +}); + +function fakeResult(completionText: string): Promise<GetNetworkCompletionsType> { + const telemetryBlob = TelemetryWithExp.createEmptyConfigForTesting(); + return Promise.resolve({ + type: 'success', + value: [fakeAPIChoice(generateUuid(), 0, completionText), Promise.resolve()], + telemetryData: mkBasicResultTelemetry(telemetryBlob), + telemetryBlob, + resultType: ResultType.Async, + }); +} diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/last.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/last.test.ts new file mode 100644 index 0000000..706502c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/last.test.ts @@ -0,0 +1,126 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import assert from 'assert'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { withInMemoryTelemetry } from '../../test/telemetry'; +import { createTextDocument } from '../../test/textDocument'; +import { CopilotCompletion } from '../copilotCompletion'; +import { ResultType } from '../ghostText'; +import { + ICompletionsLastGhostText, handleGhostTextPostInsert, + handleGhostTextShown, + handlePartialGhostTextPostInsert, + rejectLastShown, + setLastShown +} from '../last'; + +suite('Isolated LastGhostText tests', function () { + let accessor: ServicesAccessor; + let last: ICompletionsLastGhostText; + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + last = accessor.get(ICompletionsLastGhostText); + }); + + function makeCompletion(index = 0, text = 'foo', offset = 0): CopilotCompletion { + return { + uuid: 'uuid-' + index, + insertText: text, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: text.length } }, + index, + displayText: text, + offset, + uri: 'file:///test', + position: { line: 0, character: 0 }, + telemetry: TelemetryWithExp.createEmptyConfigForTesting(), + resultType: ResultType.Network, + } as CopilotCompletion; + } + + test('full completion flow: show, accept, reset', function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + const cmp = makeCompletion(1, 'full completion', 0); + handleGhostTextShown(accessor, cmp); + assert.strictEqual(last.shownCompletions.length, 1); + handleGhostTextPostInsert(accessor, cmp); + assert.strictEqual(last.shownCompletions.length, 0); + assert.strictEqual(last.position, undefined); + assert.strictEqual(last.uri, undefined); + }); + + test('partial completion flow: show, partial accept, state', function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + const cmp = makeCompletion(2, 'partial completion', 0); + handleGhostTextShown(accessor, cmp); + assert.strictEqual(last.shownCompletions.length, 1); + handlePartialGhostTextPostInsert(accessor, cmp, 7); // accept first 7 chars + assert.strictEqual(last.partiallyAcceptedLength, 7); + // State is not reset by partial accept + assert.strictEqual(last.shownCompletions.length, 1); + }); + + test('reject after show clears completions', function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + const cmp = makeCompletion(3, 'reject me', 0); + handleGhostTextShown(accessor, cmp); + assert.strictEqual(last.shownCompletions.length, 1); + rejectLastShown(accessor, 0); + assert.strictEqual(last.shownCompletions.length, 0); + }); + + test('setLastShown resets completions if position/uri changes', function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + last.shownCompletions.push(makeCompletion(4, 'baz', 0)); + const doc = createTextDocument('file:///other', 'plaintext', 1, ''); + setLastShown(accessor, doc, { line: 1, character: 1 }, ResultType.Network); + assert.strictEqual(last.shownCompletions.length, 0); + }); + + test('full acceptance sends total number of lines with telemetry', async function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + const cmp = makeCompletion(0, 'line1\nline2\nline3', 0); + handleGhostTextShown(accessor, cmp); + + const { reporter } = await withInMemoryTelemetry(accessor, () => { + handleGhostTextPostInsert(accessor, cmp); + }); + + const event = reporter.events.find(e => e.name === 'ghostText.accepted'); + assert.ok(event); + assert.strictEqual(event.measurements.numLines, 3); + }); + + test('partial acceptance for VS Code sends total number of lines accepted with telemetry', async function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + const cmp = makeCompletion(0, 'line1\nline2\nline3', 0); + handleGhostTextShown(accessor, cmp); + + const { reporter } = await withInMemoryTelemetry(accessor, () => { + handlePartialGhostTextPostInsert(accessor, cmp, 'line1'.length); + }); + + const event = reporter.events.find(e => e.name === 'ghostText.accepted'); + assert.ok(event); + assert.strictEqual(event.measurements.numLines, 1); + }); + + test('additional partial acceptance for VS Code sends total number of lines accepted with telemetry', async function () { + last.setState({ uri: 'file:///test' }, { line: 0, character: 0 }); + const cmp = makeCompletion(0, 'line1\nline2\nline3', 0); + handleGhostTextShown(accessor, cmp); + handlePartialGhostTextPostInsert(accessor, cmp, 'line1'.length); + cmp.displayText = 'line2\nline3'; // Simulate the display text being updated after accepting the first line + + const { reporter } = await withInMemoryTelemetry(accessor, () => { + handlePartialGhostTextPostInsert(accessor, cmp, 'line2'.length); + }); + + const event = reporter.events.reverse().find(e => e.name === 'ghostText.accepted'); + assert.ok(event); + assert.strictEqual(event.measurements.numLines, 2); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/multilineModel.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/multilineModel.test.ts new file mode 100644 index 0000000..466dd92 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/multilineModel.test.ts @@ -0,0 +1,456 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { MultilineModelFeatures, PromptFeatures, hasComment, requestMultilineScore } from '../multilineModel'; + +suite('multilineModel tests', function () { + this.timeout(10000); + + test('hasComment correctly identifies presence of comment for a given string, line, and language', function () { + const testCases = [ + { + string: 'def test_fn(x):\n # Print x\n print(x)', + language: 'python', + lineNumber: 0, + expected: false, + }, + { + string: 'def test_fn(x):\n # Print x\n print(x)', + language: 'python', + lineNumber: 1, + expected: true, + }, + { + string: 'def test_fn(x):\n # Print x\n print(x)', + language: 'python', + lineNumber: -2, + expected: true, + }, + { + string: '', + language: 'python', + lineNumber: 0, + expected: false, + }, + { + string: '// Comment\nconst x = 1;', + language: 'javascript', + lineNumber: 0, + expected: true, + }, + { + string: '// Comment\nconst x = 1;', + language: 'javascript', + lineNumber: 1, + expected: false, + }, + { + string: '// Comment\nconst x = 1;', + language: 'javascript', + lineNumber: 2, + expected: false, + }, + ]; + + for (const testCase of testCases) { + const { string, language, lineNumber, expected } = testCase; + assert.strictEqual(hasComment(string, lineNumber, language), expected); + } + }); + + test('PromptFeatures correctly parses prompt text', function () { + const testCases = [ + { + string: 'def test_fn(x):\n # Print x\n print(x)', + language: 'python', + length: 42, + firstLineLength: 15, + lastLineLength: 12, + lastLineRstripLength: 12, + lastLineStripLength: 8, + rstripLength: 42, + stripLength: 42, + rstripLastLineLength: 12, + rstripLastLineStripLength: 8, + secondToLastLineHasComment: true, + rstripSecondToLastLineHasComment: true, + prefixEndsWithNewline: false, + lastChar: ')', + rstripLastChar: ')', + firstChar: 'd', + lstripFirstChar: 'd', + }, + { + string: ' ', + language: 'python', + length: 1, + firstLineLength: 1, + lastLineLength: 1, + lastLineRstripLength: 0, + lastLineStripLength: 0, + rstripLength: 0, + stripLength: 0, + rstripLastLineLength: 0, + rstripLastLineStripLength: 0, + secondToLastLineHasComment: false, + rstripSecondToLastLineHasComment: false, + prefixEndsWithNewline: false, + lastChar: ' ', + rstripLastChar: '', + firstChar: ' ', + lstripFirstChar: '', + }, + { + string: '// Comment\nconst x = 1;\n', + language: 'javascript', + length: 24, + firstLineLength: 10, + lastLineLength: 12, + lastLineRstripLength: 12, + lastLineStripLength: 12, + rstripLength: 23, + stripLength: 23, + rstripLastLineLength: 12, + rstripLastLineStripLength: 12, + secondToLastLineHasComment: true, + rstripSecondToLastLineHasComment: true, + prefixEndsWithNewline: true, + lastChar: '\n', + rstripLastChar: ';', + firstChar: '/', + lstripFirstChar: '/', + }, + ]; + + for (const testCase of testCases) { + const { + string, + language, + length, + firstLineLength, + lastLineLength, + lastLineRstripLength, + lastLineStripLength, + rstripLength, + stripLength, + rstripLastLineLength, + rstripLastLineStripLength, + secondToLastLineHasComment, + rstripSecondToLastLineHasComment, + prefixEndsWithNewline, + lastChar, + rstripLastChar, + firstChar, + lstripFirstChar, + } = testCase; + const promptFeatures = new PromptFeatures(string, language); + assert.strictEqual(promptFeatures.length, length); + assert.strictEqual(promptFeatures.firstLineLength, firstLineLength); + assert.strictEqual(promptFeatures.lastLineLength, lastLineLength); + assert.strictEqual(promptFeatures.lastLineRstripLength, lastLineRstripLength); + assert.strictEqual(promptFeatures.lastLineStripLength, lastLineStripLength); + assert.strictEqual(promptFeatures.rstripLength, rstripLength); + assert.strictEqual(promptFeatures.stripLength, stripLength); + assert.strictEqual(promptFeatures.rstripLastLineLength, rstripLastLineLength); + assert.strictEqual(promptFeatures.rstripLastLineStripLength, rstripLastLineStripLength); + assert.strictEqual(promptFeatures.secondToLastLineHasComment, secondToLastLineHasComment); + assert.strictEqual(promptFeatures.rstripSecondToLastLineHasComment, rstripSecondToLastLineHasComment); + assert.strictEqual(promptFeatures.prefixEndsWithNewline, prefixEndsWithNewline); + assert.strictEqual(promptFeatures.lastChar, lastChar); + assert.strictEqual(promptFeatures.rstripLastChar, rstripLastChar); + assert.strictEqual(promptFeatures.firstChar, firstChar); + assert.strictEqual(promptFeatures.lstripFirstChar, lstripFirstChar); + } + }); + + test('MultilineModelFeatures has expected prefix and suffix features', function () { + const prefix = 'def test_fn(x):\n # Print x\n print(x)'; + const suffix = ' '; + const language = 'python'; + const prefixFeatures = { + string: 'def test_fn(x):\n # Print x\n print(x)', + language: 'python', + length: 42, + firstLineLength: 15, + lastLineLength: 12, + lastLineRstripLength: 12, + lastLineStripLength: 8, + rstripLength: 42, + stripLength: 42, + rstripLastLineLength: 12, + rstripLastLineStripLength: 8, + secondToLastLineHasComment: true, + rstripSecondToLastLineHasComment: true, + prefixEndsWithNewline: false, + lastChar: ')', + rstripLastChar: ')', + firstChar: 'd', + lstripFirstChar: 'd', + }; + const suffixFeatures = { + string: ' ', + language: 'python', + length: 1, + firstLineLength: 1, + lastLineLength: 1, + lastLineRstripLength: 0, + lastLineStripLength: 0, + rstripLength: 0, + stripLength: 0, + rstripLastLineLength: 0, + rstripLastLineStripLength: 0, + secondToLastLineHasComment: false, + rstripSecondToLastLineHasComment: false, + prefixEndsWithNewline: false, + lastChar: ' ', + rstripLastChar: '', + firstChar: ' ', + lstripFirstChar: '', + }; + const multilineFeatures = new MultilineModelFeatures(prefix, suffix, language); + assert.strictEqual(multilineFeatures.language, language); + assert.strictEqual(multilineFeatures.prefixFeatures.firstLineLength, prefixFeatures.firstLineLength); + assert.strictEqual(multilineFeatures.prefixFeatures.lastLineLength, prefixFeatures.lastLineLength); + assert.strictEqual(multilineFeatures.prefixFeatures.lastLineRstripLength, prefixFeatures.lastLineRstripLength); + assert.strictEqual(multilineFeatures.prefixFeatures.lastLineStripLength, prefixFeatures.lastLineStripLength); + assert.strictEqual(multilineFeatures.prefixFeatures.rstripLength, prefixFeatures.rstripLength); + assert.strictEqual(multilineFeatures.prefixFeatures.stripLength, prefixFeatures.stripLength); + assert.strictEqual(multilineFeatures.prefixFeatures.rstripLastLineLength, prefixFeatures.rstripLastLineLength); + assert.strictEqual( + multilineFeatures.prefixFeatures.rstripLastLineStripLength, + prefixFeatures.rstripLastLineStripLength + ); + assert.strictEqual( + multilineFeatures.prefixFeatures.secondToLastLineHasComment, + prefixFeatures.secondToLastLineHasComment + ); + assert.strictEqual( + multilineFeatures.prefixFeatures.rstripSecondToLastLineHasComment, + prefixFeatures.rstripSecondToLastLineHasComment + ); + assert.strictEqual( + multilineFeatures.prefixFeatures.prefixEndsWithNewline, + prefixFeatures.prefixEndsWithNewline + ); + assert.strictEqual(multilineFeatures.prefixFeatures.lastChar, prefixFeatures.lastChar); + assert.strictEqual(multilineFeatures.prefixFeatures.rstripLastChar, prefixFeatures.rstripLastChar); + assert.strictEqual(multilineFeatures.prefixFeatures.firstChar, prefixFeatures.firstChar); + assert.strictEqual(multilineFeatures.prefixFeatures.lstripFirstChar, prefixFeatures.lstripFirstChar); + assert.strictEqual(multilineFeatures.suffixFeatures.firstLineLength, suffixFeatures.firstLineLength); + assert.strictEqual(multilineFeatures.suffixFeatures.lastLineLength, suffixFeatures.lastLineLength); + assert.strictEqual(multilineFeatures.suffixFeatures.lastLineRstripLength, suffixFeatures.lastLineRstripLength); + assert.strictEqual(multilineFeatures.suffixFeatures.lastLineStripLength, suffixFeatures.lastLineStripLength); + assert.strictEqual(multilineFeatures.suffixFeatures.rstripLength, suffixFeatures.rstripLength); + assert.strictEqual(multilineFeatures.suffixFeatures.stripLength, suffixFeatures.stripLength); + assert.strictEqual(multilineFeatures.suffixFeatures.rstripLastLineLength, suffixFeatures.rstripLastLineLength); + assert.strictEqual( + multilineFeatures.suffixFeatures.rstripLastLineStripLength, + suffixFeatures.rstripLastLineStripLength + ); + assert.strictEqual( + multilineFeatures.suffixFeatures.secondToLastLineHasComment, + suffixFeatures.secondToLastLineHasComment + ); + assert.strictEqual( + multilineFeatures.suffixFeatures.rstripSecondToLastLineHasComment, + suffixFeatures.rstripSecondToLastLineHasComment + ); + assert.strictEqual( + multilineFeatures.suffixFeatures.prefixEndsWithNewline, + suffixFeatures.prefixEndsWithNewline + ); + assert.strictEqual(multilineFeatures.suffixFeatures.lastChar, suffixFeatures.lastChar); + assert.strictEqual(multilineFeatures.suffixFeatures.rstripLastChar, suffixFeatures.rstripLastChar); + assert.strictEqual(multilineFeatures.suffixFeatures.firstChar, suffixFeatures.firstChar); + assert.strictEqual(multilineFeatures.suffixFeatures.lstripFirstChar, suffixFeatures.lstripFirstChar); + }); + + test('MultilineModelFeatures.constructFeatures() returns correct feature array', function () { + const prefix = 'def test_fn(x):\n # Print x\n print(x)'; + const suffix = ' '; + const language = 'python'; + const prefixFeatures = { + string: 'def test_fn(x):\n # Print x\n print(x)', + language: 'python', + length: 42, + firstLineLength: 15, + lastLineLength: 12, + lastLineRstripLength: 12, + lastLineStripLength: 8, + rstripLength: 42, + stripLength: 42, + rstripLastLineLength: 12, + rstripLastLineStripLength: 8, + secondToLastLineHasComment: true, + rstripSecondToLastLineHasComment: true, + prefixEndsWithNewline: false, + lastChar: ')', + rstripLastChar: ')', + firstChar: 'd', + lstripFirstChar: 'd', + }; + const suffixFeatures = { + string: ' ', + language: 'python', + length: 1, + firstLineLength: 1, + lastLineLength: 1, + lastLineRstripLength: 0, + lastLineStripLength: 0, + rstripLength: 0, + stripLength: 0, + rstripLastLineLength: 0, + rstripLastLineStripLength: 0, + secondToLastLineHasComment: false, + rstripSecondToLastLineHasComment: false, + prefixEndsWithNewline: false, + lastChar: ' ', + rstripLastChar: '', + firstChar: ' ', + lstripFirstChar: '', + }; + const expectedNumericFeatures = [ + prefixFeatures.length, + prefixFeatures.firstLineLength, + prefixFeatures.lastLineLength, + prefixFeatures.lastLineRstripLength, + prefixFeatures.lastLineStripLength, + prefixFeatures.rstripLength, + prefixFeatures.rstripLastLineLength, + prefixFeatures.rstripLastLineStripLength, + suffixFeatures.length, + suffixFeatures.firstLineLength, + suffixFeatures.lastLineLength, + prefixFeatures.secondToLastLineHasComment ? 1 : 0, + prefixFeatures.rstripSecondToLastLineHasComment ? 1 : 0, + prefixFeatures.prefixEndsWithNewline ? 1 : 0, + ]; + const expectedLangFeatures: number[] = new Array<number>(8).fill(0); + expectedLangFeatures[5] = 1; + const expectedPrefixLastCharFeatures: number[] = new Array<number>(96).fill(0); + expectedPrefixLastCharFeatures[10] = 1; + const expectedPrefiRstripLastCharFeatures: number[] = new Array<number>(96).fill(0); + expectedPrefiRstripLastCharFeatures[10] = 1; + const expectedSuffixFirstCharFeatures: number[] = new Array<number>(96).fill(0); + expectedSuffixFirstCharFeatures[1] = 1; + const expectedSuffixLstripFirstCharFeatures: number[] = new Array<number>(96).fill(0); + expectedSuffixLstripFirstCharFeatures[0] = 1; + + const multilineFeatures = new MultilineModelFeatures(prefix, suffix, language); + const multilineFeatureArray = multilineFeatures.constructFeatures(); + // Numeric features match + assert.deepStrictEqual(multilineFeatureArray.slice(0, expectedNumericFeatures.length), expectedNumericFeatures); + // Language features match + assert.deepStrictEqual( + multilineFeatureArray.slice(expectedNumericFeatures.length, expectedNumericFeatures.length + 8), + expectedLangFeatures + ); + // Prefix last char features match + assert.deepStrictEqual( + multilineFeatureArray.slice(expectedNumericFeatures.length + 8, expectedNumericFeatures.length + 8 + 96), + expectedPrefixLastCharFeatures + ); + // Prefix rstrip last char features match + assert.deepStrictEqual( + multilineFeatureArray.slice( + expectedNumericFeatures.length + 8 + 96, + expectedNumericFeatures.length + 8 + 96 * 2 + ), + expectedPrefiRstripLastCharFeatures + ); + // Suffix first char features match + assert.deepStrictEqual( + multilineFeatureArray.slice( + expectedNumericFeatures.length + 8 + 96 * 2, + expectedNumericFeatures.length + 8 + 96 * 3 + ), + expectedSuffixFirstCharFeatures + ); + // Suffix lstrip first char features match + assert.deepStrictEqual( + multilineFeatureArray.slice( + expectedNumericFeatures.length + 8 + 96 * 3, + expectedNumericFeatures.length + 8 + 96 * 4 + ), + expectedSuffixLstripFirstCharFeatures + ); + // All features match + assert.deepStrictEqual( + multilineFeatureArray, + expectedNumericFeatures.concat( + expectedLangFeatures, + expectedPrefixLastCharFeatures, + expectedPrefiRstripLastCharFeatures, + expectedSuffixFirstCharFeatures, + expectedSuffixLstripFirstCharFeatures + ) + ); + }); + + test('requestMultilineScore() returns expected score', function () { + const testCases = [ + { + prompt: { + prefix: '// Language: javascript\nexport function(x) {', + suffix: '', + isFimEnabled: true, + }, + language: 'javascript', + score: 0.32191348, + }, + { + prompt: { + prefix: '#!/usr/bin/env python3\n# Function that adds two numbers\n', + suffix: '', + isFimEnabled: true, + }, + language: 'python', + score: 0.45744361, + }, + { + prompt: { + prefix: '#!/usr/bin/env python3\nclass Test:\n # Function that adds two numbers\n', + suffix: '', + isFimEnabled: true, + }, + language: 'python', + score: 0.40182054, + }, + { + prompt: { + prefix: '// Language: typescript\nconst testConst = ', + suffix: 'const testConst2 = 2', + isFimEnabled: true, + }, + language: 'typescript', + score: 0.45507183, + }, + { + prompt: { + prefix: '// Language: typescript\nconst testConst = ', + suffix: 'const testConst2 = 2\nconst testConst3 = 3', + isFimEnabled: true, + }, + language: 'typescript', + score: 0.45507183, + }, + { + prompt: { + prefix: '// Language: typescript\nconst testConst = \nconst testConst2 = 2\nconst testConst3 = 3 ', + suffix: '', + isFimEnabled: true, + }, + language: 'typescript', + score: 0.30417124, + }, + ]; + + for (const testCase of testCases) { + const { prompt, language, score } = testCase; + assert.strictEqual(requestMultilineScore(prompt, language).toFixed(4), score.toFixed(4)); + } + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/normalizeIndent.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/normalizeIndent.test.ts new file mode 100644 index 0000000..7acc4ff --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/normalizeIndent.test.ts @@ -0,0 +1,190 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { GhostCompletion } from '../ghostText'; +import { ITextEditorOptions, normalizeIndentCharacter } from '../normalizeIndent'; +import * as assert from 'assert'; + +suite('Leading whitespace normalization tests', function () { + test('Leading spaces are replaces with tabs', function () { + const teo: ITextEditorOptions = { + tabSize: 4, + insertSpaces: false, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: ' fun()\n yeet()', + displayText: ' fun()\n yeet()', + displayNeedsWsOffset: false, + }; + + const output = '\tfun()\n\tyeet()'; + const result = normalizeIndentCharacter(teo, completion, false); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Leading tabs are replaces with spaces', function () { + const teo: ITextEditorOptions = { + tabSize: 4, + insertSpaces: true, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: '\tfun()\n\tyeet()', + displayText: '\tfun()\n\tyeet()', + displayNeedsWsOffset: false, + }; + + const output = ' fun()\n yeet()'; + + const result = normalizeIndentCharacter(teo, completion, false); + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Leading tabs are replaces with spaces - multiple level of indents', function () { + const teo: ITextEditorOptions = { + tabSize: 2, + insertSpaces: true, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: '\tfun()\n\t\tyeet()\n\tboo()', + displayText: '\tfun()\n\t\tyeet()\n\tboo()', + displayNeedsWsOffset: false, + }; + + const output = ' fun()\n yeet()\n boo()'; + const result = normalizeIndentCharacter(teo, completion, false); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Leading spaces are replaces with tabs - multiple level of indents', function () { + const teo: ITextEditorOptions = { + tabSize: 2, + insertSpaces: false, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: ' fun()\n yeet()\n boo()', + displayText: ' fun()\n yeet()\n boo()', + displayNeedsWsOffset: false, + }; + + const output = '\tfun()\n\t\tyeet()\n\tboo()'; + const result = normalizeIndentCharacter(teo, completion, false); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Extra spaces are not dropped when replacing spaces with tabs', function () { + const teo: ITextEditorOptions = { + tabSize: 4, + insertSpaces: false, + }; + + const input = ' '.repeat(6) + 'fun()\n' + ' '.repeat(6) + ' yeet()\n' + ' '.repeat(6) + 'boo()'; + const completion: GhostCompletion = { + completionIndex: 0, + completionText: input, + displayText: input, + displayNeedsWsOffset: false, + }; + + const output = '\t fun()\n' + '\t\tyeet()\n' + '\t boo()'; + const result = normalizeIndentCharacter(teo, completion, false); + + assert.strictEqual(result.completionText, output, 'Leading whitespace normalization failed'); + assert.strictEqual(result.displayText, output, 'Leading whitespace normalization failed'); + }); + + test('Leading spaces are normalized to the tab size expected in editor in case of empty line suggestion', function () { + const teo: ITextEditorOptions = { + tabSize: 4, + insertSpaces: true, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: ' fun()\n yeet()\n boo()', + displayText: ' fun()\n yeet()\n boo()', + displayNeedsWsOffset: false, + }; + + const output = ' fun()\n yeet()\n boo()'; + const result = normalizeIndentCharacter(teo, completion, true); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Leading spaces are normalized to the tab size expected in editor in case of empty line suggestion, lot of indentation case', function () { + const teo: ITextEditorOptions = { + tabSize: 4, + insertSpaces: true, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: ' fun()\n yeet()\n boo()', + displayText: ' fun()\n yeet()\n boo()', + displayNeedsWsOffset: false, + }; + + const output = ' fun()\n yeet()\n boo()'; + const result = normalizeIndentCharacter(teo, completion, true); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Leading spaces are not normalized if ident size is same as tab size', function () { + const teo: ITextEditorOptions = { + tabSize: 2, + insertSpaces: true, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: ' fun()\n yeet()\n boo()', + displayText: ' fun()\n yeet()\n boo()', + displayNeedsWsOffset: false, + }; + + const output = ' fun()\n yeet()\n boo()'; + const result = normalizeIndentCharacter(teo, completion, true); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); + + test('Leading newlines do not trigger spurious extra indentation', function () { + const teo: ITextEditorOptions = { + tabSize: 2, + insertSpaces: true, + }; + + const completion: GhostCompletion = { + completionIndex: 0, + completionText: '\n fun()\n yeet()\n boo()', + displayText: '\n fun()\n yeet()\n boo()', + displayNeedsWsOffset: false, + }; + + const output = '\n fun()\n yeet()\n boo()'; + const result = normalizeIndentCharacter(teo, completion, true); + + assert.ok(result.completionText === output, 'Leading whitespace normalization failed'); + assert.ok(result.displayText === output, 'Leading whitespace normalization failed'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/statementTree.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/statementTree.test.ts new file mode 100644 index 0000000..bff6516 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/statementTree.test.ts @@ -0,0 +1,4577 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// WARNING: the file needs to keep space for some of the tests. So please don;t reformat. + +import assert from 'assert'; +import dedent from 'ts-dedent'; +import { StatementNode, StatementTree } from '../statementTree'; + +type StatementNodeSpec = { + startOffset: number; + endOffset?: number; + parent?: StatementNodeSpec; + children: StatementNodeSpec[]; +}; + +suite('StatementTree', function () { + test('tree with offsets includes the enclosing statements but no other statements outside the range', async function () { + await testStatementBuilding( + 'typescript', + dedent` + const ignoredStatement = 1; + + โ–ถ๏ธfunction fibonacci(n: number): number โ–ถ๏ธ{ + if (n <= 1) { + return n; + } + โ–ถ๏ธreturnโš fibonacci(n - 1) + fibonacci(n - 2);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + // Test for types of statements we want to match in supported languages and + // document the behavior of the current grammar: + + // MARK: JavaScript / TypeScript + + suite('JavaScript / Typescript', function () { + ['javascript', 'javascriptreact', 'jsx', 'typescript', 'typescriptreact'].forEach(language => { + test(`${language} is supported`, function () { + assert.strictEqual(StatementTree.isSupported(language), true); + }); + }); + + test('recognizes simple expression statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธx = 1;โ—€๏ธ + โ–ถ๏ธy = 2;โ—€๏ธ + ` + ); + }); + + test('ignores comments', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธx = 1;โ—€๏ธ + // comment + โ–ถ๏ธy = 2;โ—€๏ธ + ` + ); + }); + + test('recognizes export statements', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธexport โ–ถ๏ธconst x = 1;โ—€๏ธโ—€๏ธ`); + }); + + test('recognizes import statements', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธimport assert from 'assert';โ—€๏ธ`); + }); + + test('recognizes debugger statements', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธdebugger;โ—€๏ธ`); + }); + + test('recognizes var declarations', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธvar x = 1;โ—€๏ธ`); + }); + + test('recognizes lexical declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธconst x = 1;โ—€๏ธ + โ–ถ๏ธlet y = 2;โ—€๏ธ + ` + ); + }); + + test('recognizes single-expression if statements as', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธif (x) + โ–ถ๏ธy = 1;โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes single-expression if statements on a single line as single statements', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธif (x) y = 1;โ—€๏ธ`); + }); + + test('recognizes single-expression if / else statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธif (x) + โ–ถ๏ธy = 1;โ—€๏ธ + else + โ–ถ๏ธy = 2;โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes single-expression if / else statements on a single line as single statements', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธif (x) y = 1; else y = 2;โ—€๏ธ`); + // Since TS and JS are different grammars and the else property changed to alternative ensure we are good in JS as well. + await testStatementBuilding('javascript', `โ–ถ๏ธif (x) y = 1; else y = 2;โ—€๏ธ`); + }); + + test('recognizes if statements with blocks', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธif (x) โ–ถ๏ธ{ + โ–ถ๏ธy = 1;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes if / else statements with blocks', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธif (x) โ–ถ๏ธ{ + โ–ถ๏ธy = 1;โ—€๏ธ + }โ—€๏ธ else โ–ถ๏ธ{ + โ–ถ๏ธy = 2;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes switch statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธswitch (x) { + case 1: + โ–ถ๏ธy = true;โ—€๏ธ + default: + โ–ถ๏ธy = false;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes for statements', async function () { + // The termination expression is not it's own statement anymore. + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธfor (let i = 0; i < 10; i++) โ–ถ๏ธ{ + โ–ถ๏ธstr += ' ';โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes for...in statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธfor (const prop in object) โ–ถ๏ธ{ + โ–ถ๏ธconsole.log(prop, object[prop]);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes for...of statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธfor (const item of [1, 2, 3]) โ–ถ๏ธ{ + โ–ถ๏ธconsole.log(item);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes while statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธwhile (true) โ–ถ๏ธ{ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes do statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธdo โ–ถ๏ธ{ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธ while (true);โ—€๏ธ + ` + ); + }); + + test('recognizes try / catch / finally statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธtry โ–ถ๏ธ{ + โ–ถ๏ธthrow new Error('oops!');โ—€๏ธ + }โ—€๏ธ catch (e) โ–ถ๏ธ{ + โ–ถ๏ธconsole.error(e.message);โ—€๏ธ + }โ—€๏ธ finally โ–ถ๏ธ{ + โ–ถ๏ธconsole.log('done!');โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes with statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธwith ({x: 1}) โ–ถ๏ธ{ + โ–ถ๏ธconsole.log(x);โ—€๏ธ // 1 + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes continue statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธwhile (false) โ–ถ๏ธ{ + โ–ถ๏ธcontinue;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes return statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธfunction foo() โ–ถ๏ธ{ + โ–ถ๏ธreturn;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes labeled statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธouter: โ–ถ๏ธfor await (chunk of stream) โ–ถ๏ธ{ + โ–ถ๏ธfor (const char of chunk) โ–ถ๏ธ{ + โ–ถ๏ธif (char === '\n') + โ–ถ๏ธbreak outer;โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes statements with ternary expressions as single statements', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธlet i = featureFlag ? 0 : 1;โ—€๏ธ + ` + ); + }); + + test('recognizes function declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธfunction noop() โ–ถ๏ธ{ + // empty + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes generator function declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธfunction* values() โ–ถ๏ธ{ + โ–ถ๏ธyield 1;โ—€๏ธ + โ–ถ๏ธyield 2;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes class declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธclass Empty { + // empty + }โ—€๏ธ + ` + ); + }); + + test('recognizes class field declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธclass ConstantIdentifier { + โ–ถ๏ธreadonly id = 1โ—€๏ธ; + }โ—€๏ธ + ` + ); + }); + + test('recognizes class method declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธclass Example { + โ–ถ๏ธconstructor() โ–ถ๏ธ{ + โ–ถ๏ธthis.value = Math.random();โ—€๏ธ + }โ—€๏ธโ—€๏ธ + + โ–ถ๏ธgetValue() โ–ถ๏ธ{ + โ–ถ๏ธreturn this.value;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes class getter and setter declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธclass Example { + โ–ถ๏ธset value(newValue) โ–ถ๏ธ{ + โ–ถ๏ธthis.value = newValue;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + + โ–ถ๏ธget value() โ–ถ๏ธ{ + โ–ถ๏ธreturn this.value;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes type alias declarations', async function () { + await testStatementBuilding('typescript', `โ–ถ๏ธtype OptionalIdentifier = number | undefined;โ—€๏ธ`); + }); + + test('recognizes interface declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธinterface Vector { + x: number; + y: number; + }โ—€๏ธ + ` + ); + }); + + test('recognizes enum declarations', async function () { + await testStatementBuilding( + 'typescript', + dedent` + โ–ถ๏ธenum Direction { + North, + South, + East, + West + }โ—€๏ธ + ` + ); + }); + + test('node.isCompoundStatementType is true for splittable statements that may contain other statements', async function () { + const doc = 'if (x) { y = 1; }'; + using tree = StatementTree.create('typescript', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + const doc = 'const y = 1;'; + using tree = StatementTree.create('typescript', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, false); + }); + }); + + // MARK: Python + + suite('Python', function () { + test('python is supported', function () { + assert.strictEqual(StatementTree.isSupported('python'), true); + }); + + test('recognizes simple expression statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธx = 1โ—€๏ธ + โ–ถ๏ธy = 2โ—€๏ธ + ` + ); + }); + + test('ignores comments', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธx = 1โ—€๏ธ + # comment + โ–ถ๏ธy = 2โ—€๏ธ + ` + ); + }); + + test('recognizes import statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธimport assertโ—€๏ธ`); + }); + + test('recognizes from import statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธfrom assert import strictโ—€๏ธ`); + }); + + test('recognizes from future import statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธfrom __future__ import annotationsโ—€๏ธ`); + }); + + test('recognizes print statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธprint aโ—€๏ธ`); + }); + + test('recognizes assert statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธassert xโ—€๏ธ`); + }); + + test('recognizes return statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธdef example(): + โ–ถ๏ธโ–ถ๏ธreturn 1โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes delete statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธdel xโ—€๏ธ`); + }); + + test('recognizes raise statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธraise ValueErrorโ—€๏ธ`); + }); + + test('recognizes pass statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธdef example(): + โ–ถ๏ธโ–ถ๏ธpassโ—€๏ธโ—€๏ธโ—€๏ธ` + ); + }); + + test('recognizes break statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธwhile True: + โ–ถ๏ธโ–ถ๏ธbreakโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes continue statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธwhile True: + โ–ถ๏ธโ–ถ๏ธcontinueโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes global statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธglobal xโ—€๏ธ`); + }); + + test('recognizes nonlocal statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธdef example(): + โ–ถ๏ธโ–ถ๏ธnonlocal xโ—€๏ธโ—€๏ธโ—€๏ธ` + ); + }); + + test('recognizes exec statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธexec 'x+=1' in Noneโ—€๏ธ`); + }); + + test('recognizes statements with list comprehensions as single statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธsome_powers_of_two = [2**n for in range(1,6) if n != 5]โ—€๏ธ`); + }); + + test('recognizes statements with lamba expressions as single statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธfn = lambda x: x+1โ—€๏ธ`); + }); + + test('recognizes if statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธif x: + โ–ถ๏ธโ–ถ๏ธy = 1โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes if statements on a single line as single statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธif x: y = 1โ—€๏ธ + ` + ); + }); + + test('recognizes if / else statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธif x: + โ–ถ๏ธโ–ถ๏ธy = 1โ—€๏ธโ—€๏ธ + else: + โ–ถ๏ธโ–ถ๏ธy = 2โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes compact if / else statements as compound statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธif x: โ–ถ๏ธโ–ถ๏ธy = 1โ—€๏ธโ—€๏ธ + else: โ–ถ๏ธโ–ถ๏ธy = 2โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes if / elif / else statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธif x: + โ–ถ๏ธโ–ถ๏ธy = 1โ—€๏ธโ—€๏ธ + elif y: + โ–ถ๏ธโ–ถ๏ธy = 2โ—€๏ธโ—€๏ธ + else: + โ–ถ๏ธโ–ถ๏ธy = 3โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes statements with conditional expressions as single statements', async function () { + await testStatementBuilding('python', `โ–ถ๏ธresult = x if y else zโ—€๏ธ`); + }); + + test('recognizes for statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธfor i in range(10): + โ–ถ๏ธโ–ถ๏ธy = 1โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes for / else statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธfor line in lines: + โ–ถ๏ธโ–ถ๏ธprint lineโ—€๏ธโ—€๏ธ + else: + โ–ถ๏ธโ–ถ๏ธprint xโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes while statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธwhile x: + โ–ถ๏ธโ–ถ๏ธprint yโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes while / else statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธwhile x: + โ–ถ๏ธโ–ถ๏ธprint yโ—€๏ธโ—€๏ธ + else: + โ–ถ๏ธโ–ถ๏ธprint zโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes try / except / finally statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธtry: + โ–ถ๏ธโ–ถ๏ธx = 1โ—€๏ธโ—€๏ธ + except: + โ–ถ๏ธโ–ถ๏ธx = 2โ—€๏ธโ—€๏ธ + finally: + โ–ถ๏ธโ–ถ๏ธx = 3โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes with statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธwith open('file.txt') as f: + โ–ถ๏ธโ–ถ๏ธx = f.read()โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes function definitions', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธdef add(x, y): + โ–ถ๏ธโ–ถ๏ธreturn x + yโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes docstrings as expressions', async function () { + // this is slightly odd that the grammar gives these an expression type, + // but it is ok for the purposes of completion trimming and block + // position determination + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธdef example(): + โ–ถ๏ธโ–ถ๏ธ""" + This is a docstring. + """โ—€๏ธ + โ–ถ๏ธpassโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes class definitions', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธclass Example: + โ–ถ๏ธโ–ถ๏ธpassโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes class method definitions', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธclass Example: + โ–ถ๏ธโ–ถ๏ธdef method(self): + โ–ถ๏ธโ–ถ๏ธpassโ—€๏ธโ—€๏ธโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes decorated definitions', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธ@decorator1 + @decorator2 + โ–ถ๏ธdef example(): + โ–ถ๏ธโ–ถ๏ธpassโ—€๏ธโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes match statements', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธmatch x:โ–ถ๏ธ + case 1: + โ–ถ๏ธโ–ถ๏ธy = 1โ—€๏ธโ—€๏ธ + case 2: + โ–ถ๏ธโ–ถ๏ธy = 2โ—€๏ธโ—€๏ธ + case _: + โ–ถ๏ธโ–ถ๏ธy = 3โ—€๏ธโ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('permits type annotations on variable assignments', async function () { + await testStatementBuilding('python', `โ–ถ๏ธx: list[int] = []โ—€๏ธ`); + }); + + test('permits type annotations on functions', async function () { + await testStatementBuilding( + 'python', + dedent` + โ–ถ๏ธdef example(x: int) -> int: + โ–ถ๏ธโ–ถ๏ธreturn x + 1โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('permits type aliases but omits the type keyword from the statement', async function () { + // this is to document the behavior of the current grammar + // type alias is not supported in 0.23 Python grammar. Results in no statement. + await testStatementBuilding('python', `type Vector = list[float]`); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + const doc = 'y = 1'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, false); + }); + + test('node.isCompoundStatementType is true for if statements', async function () { + const doc = 'if x:\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for for statements', async function () { + const doc = 'for i in range(10):\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for while statements', async function () { + const doc = 'while x:\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for try statements', async function () { + const doc = 'try:\n\tpass\nexcept:\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for with statements', async function () { + const doc = 'with open("file.txt") as f:\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for function definition statements', async function () { + const doc = 'def example():\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for class definition statements', async function () { + const doc = 'class Example:\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for decorated definition statements', async function () { + const doc = '@decorator\ndef example():\n\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is true for match statements', async function () { + const doc = 'match x:\n\tcase 1:\n\t\tpass'; + using tree = StatementTree.create('python', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + }); + + // MARK: Go + + suite('Go', function () { + test('go is supported', function () { + assert.strictEqual(StatementTree.isSupported('go'), true); + }); + + test('recognizes package clauses', async function () { + await testStatementBuilding('go', `โ–ถ๏ธpackage mainโ—€๏ธ`); + }); + + test('recognizes function declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc example() โ–ถ๏ธ{}โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes method declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc (self Document) GetLine(n int) โ–ถ๏ธ{}โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes import declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธimport "fmt"โ—€๏ธ + ` + ); + }); + + test('recognizes grouped import declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธimport ( + "fmt" + "os + )โ—€๏ธ + ` + ); + }); + + test('ignores comments', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + // comment + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('ignores block comments', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + /* + * Comment + */ + โ–ถ๏ธfunc main() โ–ถ๏ธ{}โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes single constant declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธconst zero = 0โ—€๏ธ + ` + ); + }); + + test('recognizes grouped constant declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธconst ( + zero = 0 + one = 1 + )โ—€๏ธ + ` + ); + }); + + test('recognizes var declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธvar counter = 0โ—€๏ธ + ` + ); + }); + + test('recognizes type declarations', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธtype a bโ—€๏ธ + ` + ); + }); + + test('recognizes simple expression statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธx := 1โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes return statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธreturnโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes go statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธgo f()โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes defer statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธdefer f()โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes if statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธif a โ–ถ๏ธ{ + โ–ถ๏ธbโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes if statements with an initializer', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธif b := a(); b < 0 โ–ถ๏ธ{ + โ–ถ๏ธb *= -1โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes if / else statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธif a โ–ถ๏ธ{ + โ–ถ๏ธb()โ—€๏ธ + }โ—€๏ธ else โ–ถ๏ธ{ + โ–ถ๏ธc()โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes simple for statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธfor โ–ถ๏ธ{ + โ–ถ๏ธa()โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes for statements with conditions', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + โ–ถ๏ธimport "fmt"โ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธfor i:= 0; i < 10; i++ โ–ถ๏ธ{ + โ–ถ๏ธfmt.Println(i)โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes expression switch statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + โ–ถ๏ธimport "fmt"โ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธswitch a { + case 1: + โ–ถ๏ธbโ—€๏ธ + case 2: + โ–ถ๏ธcโ—€๏ธ + default: + โ–ถ๏ธdโ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes type switch statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc debug(i interface{}) โ–ถ๏ธ{ + โ–ถ๏ธswitch v := i.(type) { + case int: + โ–ถ๏ธfmt.Printf("%v is an integer", v)โ—€๏ธ + case string: + โ–ถ๏ธfmt.Printf("%q is a string", v)โ—€๏ธ + default: + โ–ถ๏ธfmt.Printf("%T is unknown", v)โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes select statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc demux(a chan string, b chan string) โ–ถ๏ธ{ + โ–ถ๏ธselect { + case msg := <-a: + โ–ถ๏ธdispatch(msg)โ—€๏ธ + case msg := <-b: + โ–ถ๏ธdispatch(msg)โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes labeled statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธstart: + โ–ถ๏ธa()โ—€๏ธโ—€๏ธ + โ–ถ๏ธb()โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes fallthrough statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธswitch i { + case 0: + โ–ถ๏ธfallthroughโ—€๏ธ + default: + โ–ถ๏ธf(i)โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes break statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธswitch i { + case 0: + โ–ถ๏ธbreakโ—€๏ธ + default: + โ–ถ๏ธf(i)โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes continue statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธfor i := 0; i < 10; i++ โ–ถ๏ธ{ + โ–ถ๏ธif i == 0 โ–ถ๏ธ{ + โ–ถ๏ธcontinueโ—€๏ธ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธf(i)โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes goto statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธgoto endโ—€๏ธ + โ–ถ๏ธend: + โ–ถ๏ธreturnโ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes nested blocks', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc main() โ–ถ๏ธ{ + โ–ถ๏ธ{ + โ–ถ๏ธa()โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes empty statements', async function () { + await testStatementBuilding( + 'go', + dedent` + โ–ถ๏ธpackage mainโ—€๏ธ + + โ–ถ๏ธfunc noop() โ–ถ๏ธ{ + โ–ถ๏ธ;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + await assertStatementIsNotCompoundType(dedent` + package main + + func main() { + โšx := 1 + } + `); + }); + + test('node.isCompoundStatementType is true for function declarations', async function () { + await assertStatementIsCompoundType(dedent` + package main + + โšfunc main() {} + `); + }); + + test('node.isCompoundStatementType is true for method declarations', async function () { + await assertStatementIsCompoundType(dedent` + package main + + โšfunc (self Document) GetLine (n int) {} + `); + }); + + test('node.isCompoundStatementType is true for if statements', async function () { + await assertStatementIsCompoundType(dedent` + package main + + func main() { + โšif a { + b + } + } + `); + }); + + test('node.isCompoundStatementType is true for for statements', async function () { + await assertStatementIsCompoundType(dedent` + package main + + func main() { + โšfor i := 0; i < 10; i++ { + a() + } + } + `); + }); + + test('node.isCompoundStatementType is true for expression switch statements', async function () { + await assertStatementIsCompoundType(dedent` + package main + + func main() { + โšswitch a { + case 1: + b + default: + c + } + } + `); + }); + + test('node.isCompoundStatementType is true for type switch statements', async function () { + await assertStatementIsCompoundType(dedent` + package main + + func f(i interface{}) { + โšswitch v := i.(type) { + case int: + b + default: + c + } + } + `); + }); + + test('node.isCompoundStatementType is true for select statements', async function () { + await assertStatementIsCompoundType(dedent` + package main + + func demux(a chan string, b chan string) { + โšselect { + case msg := <-a: + dispatch(msg) + case msg := <-b: + dispatch(msg) + } + } + `); + }); + + async function testStatementIsCompoundType(text: string, expectedResult: boolean) { + const posIndicator = 'โš'; + const offset = text.indexOf(posIndicator); + const doc = text.replace(posIndicator, ''); + using tree = StatementTree.create('go', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(offset + 1); + + assert.ok(statement, `Statement not found at offset ${offset}`); + assert.strictEqual(statement.isCompoundStatementType, expectedResult); + } + + async function assertStatementIsCompoundType(text: string) { + await testStatementIsCompoundType(text, true); + } + + async function assertStatementIsNotCompoundType(text: string) { + await testStatementIsCompoundType(text, false); + } + }); + + // MARK: Php + suite('PHP', function () { + test('Php is supported', function () { + assert.strictEqual(StatementTree.isSupported('php'), true); + }); + + test('recognizes simple expressions', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธecho "hello";โ—€๏ธ + โ–ถ๏ธ$b = $a = 5;โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes named if statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธif (1 == 2) โ–ถ๏ธ{ + โ–ถ๏ธecho "hello";โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes if statements with else', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธif (1 == 2) โ–ถ๏ธ{ + โ–ถ๏ธecho "hello";โ—€๏ธ + }โ—€๏ธ else โ–ถ๏ธ{ + โ–ถ๏ธecho "world";โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes if statements with else if', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธif (1 == 2) โ–ถ๏ธ{ + โ–ถ๏ธecho "hello";โ—€๏ธ + }โ—€๏ธ elseif (1 == 3) โ–ถ๏ธ{ + โ–ถ๏ธecho "world";โ—€๏ธ + }โ—€๏ธ else โ–ถ๏ธ{ + โ–ถ๏ธecho "foo";โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes switch statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธswitch ($a) { + case 1: + โ–ถ๏ธecho "hello";โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + case 2: + โ–ถ๏ธecho "world";โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + default: + โ–ถ๏ธecho "foo";โ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes while statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธwhile (true) โ–ถ๏ธ{ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes do statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธdo โ–ถ๏ธ{ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธ while (true);โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes for statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธfor ($i = 0; $i < 10; $i++) โ–ถ๏ธ{ + โ–ถ๏ธ$str += ' ';โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes foreach statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธforeach ($arr as $key => $value) โ–ถ๏ธ{ + โ–ถ๏ธecho $key;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes try statements', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธtry โ–ถ๏ธ{ + โ–ถ๏ธthrow new Exception();โ—€๏ธ + }โ—€๏ธ catch (Exception $e) โ–ถ๏ธ{ + โ–ถ๏ธecho $e;โ—€๏ธ + }โ—€๏ธ finally โ–ถ๏ธ{ + โ–ถ๏ธecho "done";โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes function declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธfunction example($arg_1) โ–ถ๏ธ{ + โ–ถ๏ธecho "hello";โ—€๏ธ + โ–ถ๏ธreturn $retval;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ?> + ` + ); + }); + + test('recognizes class declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธclass Example { + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes class method declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธclass Example { + โ–ถ๏ธpublic function example($arg_1) โ–ถ๏ธ{ + โ–ถ๏ธecho "hello";โ—€๏ธ + โ–ถ๏ธreturn $retval;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes class field declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธclass Example { + โ–ถ๏ธpublic $field_1;โ—€๏ธ + โ–ถ๏ธprivate $field_2;โ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes class constant declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธclass Example { + โ–ถ๏ธconst EXAMPLE = 1;โ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes class interface and trait uses', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธclass Example extends BaseClass implements Interface1, Interface2 { + โ–ถ๏ธuse Trait1, Trait2;โ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes interface declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธinterface Example { + โ–ถ๏ธpublic function example($arg_1);โ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes trait declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธtrait Example { + โ–ถ๏ธpublic function example($arg_1) โ–ถ๏ธ{ + โ–ถ๏ธecho "hello";โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes namespace declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธnamespace Example;โ—€๏ธ + ?> + ` + ); + }); + + test('recognizes namespace use declarations', async function () { + await testStatementBuilding( + 'php', + dedent` + <?php + โ–ถ๏ธuse Example\\ExampleClass;โ—€๏ธ + ?> + ` + ); + }); + + test('node.isCompoundStatementType is true for splittable statements that may contain other statements', async function () { + const doc = dedent`<?php + if (true) + { + $foo = 1; + } + ?>`; + using tree = StatementTree.create('php', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(6); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + const doc = dedent`<?php + $foo = 1; + ?>`; + using tree = StatementTree.create('php', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(6); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, false); + }); + }); + + // MARK: Ruby + suite('Ruby', function () { + test('ruby is supported', function () { + assert.strictEqual(StatementTree.isSupported('ruby'), true); + }); + + test('recognizes simple expression statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธx = 1โ—€๏ธ + โ–ถ๏ธy = 2โ—€๏ธ + ` + ); + }); + + test('ignores comments', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธx = 1โ—€๏ธ + # comment + โ–ถ๏ธy = 2โ—€๏ธ + ` + ); + }); + + test('recognizes if statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธif โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes if / else statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธif โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + else + โ–ถ๏ธy = 2โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes if / elsif / else statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธif โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + elsif โ–ถ๏ธyโ—€๏ธ + โ–ถ๏ธy = 2โ—€๏ธ + else + โ–ถ๏ธy = 3โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes unless statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธunless โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes unless / else statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธunless โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + else + โ–ถ๏ธy = 2โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes unless / elsif / else statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธunless โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + elsif โ–ถ๏ธyโ—€๏ธ + โ–ถ๏ธy = 2โ—€๏ธ + else + โ–ถ๏ธy = 3โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes if modifier statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธโ–ถ๏ธx = 1โ—€๏ธ if yโ—€๏ธ + ` + ); + }); + + test('recognizes unless modifier statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธโ–ถ๏ธx = 1โ—€๏ธ unless yโ—€๏ธ + ` + ); + }); + + test('recognizes range statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธx = 1..10โ—€๏ธ + ` + ); + }); + + test('recognizes case statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธcase โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธwhen 1 + โ–ถ๏ธy = 1โ—€๏ธโ—€๏ธ + โ–ถ๏ธwhen 2 + โ–ถ๏ธy = 2โ—€๏ธโ—€๏ธ + else + โ–ถ๏ธy = 3โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes for statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธfor i in 1..10 do + โ–ถ๏ธy = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes while statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธwhile โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes until statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธuntil โ–ถ๏ธxโ—€๏ธ + โ–ถ๏ธy = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes loop modifier statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธโ–ถ๏ธsleepโ—€๏ธ while idleโ—€๏ธ + โ–ถ๏ธโ–ถ๏ธsleepโ—€๏ธ until idleโ—€๏ธ + ` + ); + }); + + test('recognizes begin / rescue / else / ensure statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธbegin + โ–ถ๏ธx = 1โ—€๏ธ + rescue + โ–ถ๏ธx = 2โ—€๏ธ + else + โ–ถ๏ธx = 3โ—€๏ธ + ensure + โ–ถ๏ธx = 4โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes begin statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธBEGIN { + โ–ถ๏ธx = 1โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes end statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธEND { + โ–ถ๏ธx = 1โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes class definitions', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธclass Example < Base + โ–ถ๏ธx = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes class definitions with methods', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธclass Example < Base + โ–ถ๏ธdef method + โ–ถ๏ธx = 1โ—€๏ธ + endโ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes module definitions', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธmodule Example + โ–ถ๏ธx = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes module definitions with methods', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธmodule Example + โ–ถ๏ธdef method + โ–ถ๏ธx = 1โ—€๏ธ + endโ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes def statements', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธdef example + โ–ถ๏ธx = 1โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('recognizes method invocation with a block argument,', async function () { + await testStatementBuilding( + 'ruby', + dedent` + โ–ถ๏ธsomeArray.select do |item| + โ–ถ๏ธitem %2 == 0โ—€๏ธ + endโ—€๏ธ + ` + ); + }); + + test('node.isCompoundStatementType is true for splittable statements that may contain other statements', async function () { + const doc = dedent` + if x + y = 1 + end + + case x + when x + y = 1 + end + + while x + y = 1 + end + + until x + y = 1 + end + + for x in y + y = 1 + end + + begin + y = 1 + rescue + y = 1 + else + y = 1 + ensure + y = 1 + end + + class X + y = 1 + end + + module X + y = 1 + end + + def x + y = 1 + end + `; + using tree = StatementTree.create('ruby', doc, 0, doc.length); + + await tree.build(); + const if_statement = tree.statementAt(1); + const case_statement = tree.statementAt(20); + const while_statement = tree.statementAt(68); + const until_statement = tree.statementAt(107); + const for_statement = tree.statementAt(146); + const begin_statement = tree.statementAt(145); + const class_statement = tree.statementAt(191); + const module_statement = tree.statementAt(214); + const def_statement = tree.statementAt(238); + + assert.ok(if_statement); + assert.strictEqual(if_statement.isCompoundStatementType, true); + assert.ok(case_statement); + assert.strictEqual(case_statement.isCompoundStatementType, true); + assert.ok(while_statement); + assert.strictEqual(while_statement.isCompoundStatementType, true); + assert.ok(until_statement); + assert.strictEqual(until_statement.isCompoundStatementType, true); + assert.ok(for_statement); + assert.strictEqual(for_statement.isCompoundStatementType, true); + assert.ok(begin_statement); + assert.strictEqual(begin_statement.isCompoundStatementType, true); + assert.ok(class_statement); + assert.strictEqual(class_statement.isCompoundStatementType, true); + assert.ok(module_statement); + assert.strictEqual(module_statement.isCompoundStatementType, true); + assert.ok(def_statement); + assert.strictEqual(def_statement.isCompoundStatementType, true); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + const doc = 'x = 1'; + using tree = StatementTree.create('ruby', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(1); + + assert.ok(statement); + assert.strictEqual(statement.isCompoundStatementType, false); + }); + }); + + // MARK: Java + + suite('Java', function () { + test('java is supported', function () { + assert.strictEqual(StatementTree.isSupported('java'), true); + }); + + test('recognizes blocks', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class BlockSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธ{}โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes assert statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class AssertSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint x = 10;โ—€๏ธ + โ–ถ๏ธassert x > 0 : "x should be positive";โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes break statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class BreakSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธfor (int i = 0; i < 10; i++) โ–ถ๏ธ{ + โ–ถ๏ธif (i == 5) โ–ถ๏ธ{ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes continue statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class ContinueSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธfor (int i = 0; i < 10; i++) โ–ถ๏ธ{ + โ–ถ๏ธif (i == 5) โ–ถ๏ธ{ + โ–ถ๏ธcontinue;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes do statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class DoWhileSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint i = 0;โ—€๏ธ + โ–ถ๏ธdo โ–ถ๏ธ{ + โ–ถ๏ธif (i == 5) โ–ถ๏ธ{ + โ–ถ๏ธcontinue;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธi++;โ—€๏ธ + }โ—€๏ธ while (i < 10);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes for-each (enhanced_for) statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class ForEachSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint[] numbers = {1, 2, 3, 4, 5};โ—€๏ธ + โ–ถ๏ธfor (int n : numbers) โ–ถ๏ธ{ + โ–ถ๏ธif (n == 5) โ–ถ๏ธ{ + โ–ถ๏ธcontinue;โ—€๏ธ + }โ—€๏ธโ—€๏ธ๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes simple expression statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class SimpleExpressionSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint x = 1;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes for statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class ForSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธfor (int i = 0; i < 10; i++) โ–ถ๏ธ{ + โ–ถ๏ธint x = i;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes if statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class IfSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint number = 1;โ—€๏ธ + โ–ถ๏ธif (number > 0) โ–ถ๏ธ{ + โ–ถ๏ธnumber++;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes labeled statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class LabelSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธmyLabel: โ–ถ๏ธ{ + โ–ถ๏ธint x = 1;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes local variable declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class LocalVariableSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint x = 1;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes return statement', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class ReturnSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint number = ReturnSample.add(5, 10);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic static int add(int a, int b) โ–ถ๏ธ{ + โ–ถ๏ธreturn a + b;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes switch statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class SwitchSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint test = 1;โ—€๏ธ + โ–ถ๏ธswitch (test) { + case 0: + โ–ถ๏ธSystem.out.println("The number is one.");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + case 1: + โ–ถ๏ธSystem.out.println("The number is zero.");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + default: + โ–ถ๏ธSystem.out.println("The number is not zero or one.");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes synchronized statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class SynchronizedSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint counter = 0;โ—€๏ธ + โ–ถ๏ธsynchronized (ReturnSample.class) โ–ถ๏ธ{ + โ–ถ๏ธcounter++;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes throw statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class ThrowSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธthrow new RuntimeException("This is a runtime exception");โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes try statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class TrySample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธtry โ–ถ๏ธ{ + โ–ถ๏ธint result = 10 / 0;โ—€๏ธ + }โ—€๏ธ catch (ArithmeticException e) โ–ถ๏ธ{ + โ–ถ๏ธSystem.out.println("Cannot divide by zero");โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes try with resources statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class TrySample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธtry (BufferedReader br = new BufferedReader()) โ–ถ๏ธ{ + โ–ถ๏ธint result = 10 / 0;โ—€๏ธ + }โ—€๏ธ catch (ArithmeticException e) โ–ถ๏ธ{ + โ–ถ๏ธSystem.out.println("Cannot divide by zero");โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes enum declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class EnumSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธpublic enum Day { + MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes import declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธimport java.util.List;โ—€๏ธ + โ–ถ๏ธpublic class ImportSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes interface declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic interface Animal { + โ–ถ๏ธvoid makeSound();โ—€๏ธ + }โ—€๏ธ + โ–ถ๏ธpublic class InterfaceSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes method declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class MethodSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic static void add(int a, int b) โ–ถ๏ธ{ + โ–ถ๏ธint sum = a + b;โ—€๏ธ + โ–ถ๏ธSystem.out.println("Sum: " + sum);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes field declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class InterfaceSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic static int x = 0;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes compact constructor declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic record Person(String firstName, String lastName) { + โ–ถ๏ธpublic Person โ–ถ๏ธ{ + โ–ถ๏ธfirstName = firstName;โ—€๏ธ + โ–ถ๏ธlastName = lastName;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes class declaration inside a class body', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class OuterSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic class InnerSample { + โ–ถ๏ธpublic static void innerMethod() โ–ถ๏ธ{ + โ–ถ๏ธint x = 0;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes interface declaration inside a class body', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class OuterSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic interface InnerInterface { + โ–ถ๏ธvoid innerMethod();โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes annotation type declaration inside a class body', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class AnnotateSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic @interface MyAnnotation { + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes enum declarations inside a class body', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class EnumClassSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic enum Day { + MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes static initializer inside a class body', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class StaticInitClassSample { + โ–ถ๏ธstatic int count;โ—€๏ธ + โ–ถ๏ธstatic โ–ถ๏ธ{ + โ–ถ๏ธcount = 100;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes constructor declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class ConstructorSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + โ–ถ๏ธpublic class MyClass { + โ–ถ๏ธpublic MyClass() { + โ–ถ๏ธint x = 0;โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes record declarations', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic record Point(int x, int y) {}โ—€๏ธ + โ–ถ๏ธpublic class RecordSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes ternary statements as one line', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class RecordSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint x = 5;โ—€๏ธ + โ–ถ๏ธint y = (x == 5) ? 0 : 1;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes single line if statements as one statement', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class SingleLineIfSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint x = 5;โ—€๏ธ + โ–ถ๏ธint y = 10;โ—€๏ธ + โ–ถ๏ธif (x == 5) y = 0;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes single line if else statements with blocks as multiple statements', async function () { + await testStatementBuilding( + 'java', + dedent` + โ–ถ๏ธpublic class SingleLineIfSample { + โ–ถ๏ธpublic static void main(String[] args) โ–ถ๏ธ{ + โ–ถ๏ธint x = 5;โ—€๏ธ + โ–ถ๏ธint y = 10;โ—€๏ธ + โ–ถ๏ธif (x == 5) โ–ถ๏ธ{ โ–ถ๏ธy = 0;โ—€๏ธ }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('node.isCompoundStatementType is true for splittable block statements', async function () { + await assertStatementIsCompoundType(dedent` + { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is true for splittable do statements', async function () { + await assertStatementIsCompoundType(dedent` + do { + int x = 1; + } while (true);`); + }); + + test('node.isCompoundStatementType is true for splittable enhanced for statements', async function () { + await assertStatementIsCompoundType(dedent` + for (int n : numbers) { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is true for splittable for statements', async function () { + await assertStatementIsCompoundType(dedent` + for (int i = 0; i < 10; i++) { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is true for splittable labeled statements', async function () { + await assertStatementIsCompoundType(dedent` + myLabel: { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is true for splittable switch expression', async function () { + await assertStatementIsCompoundType(dedent` + switch (test) { + case 0: + System.out.println("The number is one."); + break; + }`); + }); + + test('node.isCompoundStatementType is true for splittable synchronized statement', async function () { + await assertStatementIsCompoundType(dedent` + synchronized (ReturnSample.class) { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is true for splittable try statement', async function () { + await assertStatementIsCompoundType(dedent` + try { + int result = 10 / 0; + } catch (ArithmeticException e) { + System.out.println("Cannot divide by zero"); + }`); + }); + + test('node.isCompoundStatementType is true for splittable try with resources statement', async function () { + await assertStatementIsCompoundType(dedent` + try (BufferedReader br = new BufferedReader(new FileReader("file.txt"))) { + int result = 10 / 0; + } catch (ArithmeticException e) { + System.out.println("Cannot divide by zero"); + }`); + }); + + test('node.isCompoundStatementType is true for splittable while statement', async function () { + await assertStatementIsCompoundType(dedent` + while (true) { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is true for splittable interface declaration', async function () { + await assertStatementIsCompoundType(dedent` + public interface InnerInterface { + void innerMethod(); + }`); + }); + + test('node.isCompoundStatementType is true for splittable method declaration', async function () { + await assertStatementIsCompoundType(dedent` + public static void add(int a, int b) { + int sum = a + b; + }`); + }); + + test('node.isCompoundStatementType is true for splittable constructor declaration', async function () { + await assertStatementIsCompoundType(dedent` + class MyClass { + โšpublic MyClass() { + int x = 0; + } + }`); + }); + + test('node.isCompoundStatementType is true for splittable compact constructor declaration', async function () { + await assertStatementIsCompoundType(dedent` + public record Person(String firstName, String lastName) { + โšpublic Person { + firstName = firstName; + lastName = lastName; + } + }`); + }); + + test('node.isCompoundStatementType is true for splittable class declaration', async function () { + await assertStatementIsCompoundType(dedent` + class MyClass { + public MyClass() { + int x = 0; + } + }`); + }); + + test('node.isCompoundStatementType is true for splittable annotation type declaration', async function () { + await assertStatementIsCompoundType(dedent` + public @interface MyAnnotation { + void myMethod(); + }`); + }); + + test('node.isCompoundStatementType is true for splittable static initializer', async function () { + await assertStatementIsCompoundType(dedent` + public class StaticInitClassSample { + static int count + โšstatic + { + count = 100; + } + }`); + }); + + test('node.isCompoundStatementType is true for splittable if statements', async function () { + await assertStatementIsCompoundType(dedent` + if (true) { + int x = 1; + }`); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + await assertStatementIsNotCompoundType('int x = 1;'); + }); + + async function testStatementIsCompoundType(text: string, expectedResult: boolean) { + const posIndicator = 'โš'; + const offset = text.indexOf(posIndicator); + const doc = text.replace(posIndicator, ''); + using tree = StatementTree.create('java', doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(offset + 1); + + assert.ok(statement, `Statement not found at offset ${offset}`); + assert.strictEqual(statement.isCompoundStatementType, expectedResult); + } + + async function assertStatementIsCompoundType(text: string) { + await testStatementIsCompoundType(text, true); + } + + async function assertStatementIsNotCompoundType(text: string) { + await testStatementIsCompoundType(text, false); + } + }); + + // MARK: C# + + suite('C#', function () { + test('csharp is supported', function () { + assert.strictEqual(StatementTree.isSupported('csharp'), true); + }); + + test('recognizes extern alias directives', async function () { + await testStatementBuilding('csharp', `โ–ถ๏ธextern alias Example;โ—€๏ธ`); + }); + + test('recognizes using directives', async function () { + await testStatementBuilding('csharp', `โ–ถ๏ธusing System;โ—€๏ธ`); + }); + + test('recognizes global attributes', async function () { + await testStatementBuilding('csharp', `โ–ถ๏ธ[assembly: AssemblyTitle("Example")]โ—€๏ธ`); + }); + + test('recognizes top-level pre-processor directives', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธ#if WIN32 + โ–ถ๏ธstring os = "Win32";โ—€๏ธ + #elif MACOS + โ–ถ๏ธstring os = "MacOS";โ—€๏ธ + #else + โ–ถ๏ธstring os = "Linux";โ—€๏ธ + #endifโ—€๏ธ + ` + ); + }); + + test('recognizes file-scoped namespace declarations', async function () { + await testStatementBuilding('csharp', `โ–ถ๏ธnamespace Example;โ—€๏ธ`); + }); + + test('recognizes namespace declarations', async function () { + await testStatementBuilding('csharp', `โ–ถ๏ธnamespace Example { }โ—€๏ธ`); + }); + + test('recognizes top-level statements', async function () { + await testStatementBuilding('csharp', `โ–ถ๏ธConsole.WriteLine("example");โ—€๏ธ`); + }); + + test('recognizes enum declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธenum Direction + { + North, + South, + East, + West + }โ—€๏ธ + ` + ); + }); + + test('recognizes class declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + }โ—€๏ธ + ` + ); + }); + + test('recognizes struct declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธstruct Example + { + }โ—€๏ธ + ` + ); + }); + + test('recognizes record declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธrecord Example + { + }โ—€๏ธ + ` + ); + }); + + test('recognizes interface declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธinterface Example + { + }โ—€๏ธ + ` + ); + }); + + test('recognizes fields', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธbool flag = true;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes event fields', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธevent EventHandler onEvent;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes properties', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธint Len + { + โ–ถ๏ธget โ–ถ๏ธ{ โ–ถ๏ธreturn _len;โ—€๏ธ }โ—€๏ธโ—€๏ธ + โ–ถ๏ธset โ–ถ๏ธ{ โ–ถ๏ธ_len = value;โ—€๏ธ }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes automatic properties', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธint Len { โ–ถ๏ธget;โ—€๏ธ โ–ถ๏ธset;โ—€๏ธ }โ—€๏ธ + โ–ถ๏ธint Capacity { โ–ถ๏ธget;โ—€๏ธ โ–ถ๏ธinit;โ—€๏ธ }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes properties with initial values', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธint Len { โ–ถ๏ธget;โ—€๏ธ } = 0;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes properties with an arrow expression', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธint Area => _width * _height;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes event declarations with add / remove functions', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธevent EventHandler onEvent + { + โ–ถ๏ธadd โ–ถ๏ธ{ โ–ถ๏ธsomeWork();โ—€๏ธ }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes methods', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes constructors', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธExample() + โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes destructors', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธ~Example() + โ–ถ๏ธ{ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes indexers', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธint this[int index]() + { + โ–ถ๏ธget โ–ถ๏ธ{ โ–ถ๏ธreturn _items[index];โ—€๏ธ }โ—€๏ธโ—€๏ธ + โ–ถ๏ธset โ–ถ๏ธ{ โ–ถ๏ธ_items[index] = value;โ—€๏ธ }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes operators', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธExample operator +(Example e) โ–ถ๏ธ{ โ–ถ๏ธreturn new Example();โ—€๏ธ }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes conversion operators', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธexplicit operator int(Example e) โ–ถ๏ธ{ โ–ถ๏ธreturn 0;โ—€๏ธ }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes delegates', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธdelegate void Action();โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes block statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine("example");โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes break statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธfor (;;) โ–ถ๏ธ{ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes expression statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธx = y * 4 + 2;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes checked statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธuint i = uint.MaxValue;โ—€๏ธ + โ–ถ๏ธchecked + โ–ถ๏ธ{ + โ–ถ๏ธi += 10;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes do statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธint i = 0;โ—€๏ธ + โ–ถ๏ธdo + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine(i);โ—€๏ธ + โ–ถ๏ธi++;โ—€๏ธ + }โ—€๏ธ while (i < 10);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes empty statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธ;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes unsafe statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธunsafe + โ–ถ๏ธ{ + โ–ถ๏ธint numbers = [1, 2, 3];โ—€๏ธ + โ–ถ๏ธint* p = numbers;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes fixed statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธunsafe + โ–ถ๏ธ{ + โ–ถ๏ธint numbers = [1, 2, 3];โ—€๏ธ + โ–ถ๏ธfixed (int* p = numbers) + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine(*p);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes for statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธfor (int i = 0; i < 5; i++) + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine(i);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes return statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธreturn;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes lock statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธlock (x) + โ–ถ๏ธ{ + // do work + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes yield statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธIEnumerable<int> Odds(int through) + โ–ถ๏ธ{ + โ–ถ๏ธfor (int i = 1; i <= through; i += 2) + โ–ถ๏ธ{ + โ–ถ๏ธyield return i;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes switch statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Diagnostics(int a, int b) + โ–ถ๏ธ{ + โ–ถ๏ธswitch ((a, b)) + { + case (> 0, > 0) when a == b: + โ–ถ๏ธConsole.WriteLine("Values are equal");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + case (> 0, > 0): + โ–ถ๏ธConsole.WriteLine("Both values are positive");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + default: + โ–ถ๏ธConsole.WriteLine("One or more values are not positive");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธ + }โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes throw statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธthrow new Exception("Error occurred");โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes try / catch / finally statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธtry + โ–ถ๏ธ{ + โ–ถ๏ธthrow new Exception("Error occurred");โ—€๏ธ + }โ—€๏ธ + catch (Exception e) + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine(e.Message);โ—€๏ธ + }โ—€๏ธ + finally + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine("Done");โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes using statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid ReadFile(string path) + โ–ถ๏ธ{ + โ–ถ๏ธusing var file = new StreamReader(path);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes foreach statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid PrintAll(List<int> numbers) + โ–ถ๏ธ{ + โ–ถ๏ธforeach (var number in numbers) + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine(number);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes goto and labeled statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธgoto End;โ—€๏ธ + + โ–ถ๏ธEnd: + โ–ถ๏ธreturn;โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes if / else statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธbool IsEven(int number) + โ–ถ๏ธ{ + โ–ถ๏ธif (number % 2 == 0) + โ–ถ๏ธ{ + โ–ถ๏ธreturn true;โ—€๏ธ + }โ—€๏ธ + else + โ–ถ๏ธ{ + โ–ถ๏ธreturn false;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('collapses single-line if statements without braces', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run(bool flag) + โ–ถ๏ธ{ + โ–ถ๏ธif (flag) return;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes while statements', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid PrintTimes(string message, int times) + โ–ถ๏ธ{ + โ–ถ๏ธint i = 0;โ—€๏ธ + โ–ถ๏ธwhile (i < times) + โ–ถ๏ธ{ + โ–ถ๏ธConsole.WriteLine(message);โ—€๏ธ + โ–ถ๏ธi++;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes local variable declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธint x = 10;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes local function declarations', async function () { + await testStatementBuilding( + 'csharp', + dedent` + โ–ถ๏ธclass Example + { + โ–ถ๏ธvoid Run() + โ–ถ๏ธ{ + โ–ถ๏ธvoid LocalFunction() โ–ถ๏ธ{ โ–ถ๏ธConsole.WriteLine("Hello from local function!");โ—€๏ธ }โ—€๏ธโ—€๏ธ + โ–ถ๏ธLocalFunction();โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('node.isCompoundStatementType is false for un-splittable statements', async function () { + await assertStatementIsNotCompoundType(dedent` + class Example + { + static void Main() + { + โšint x = 1; + } + } + `); + }); + + test('node.isCompoundStatementType is true for class declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšclass Example + { + } + `); + }); + + test('node.isCompoundStatementType is true for struct declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšstruct Example + { + } + `); + }); + + test('node.isCompoundStatementType is true for interface declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšinterface Example + { + } + `); + }); + + test('node.isCompoundStatementType is true for method declarations', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + โšvoid Run() + { + } + } + `); + }); + + test('node.isCompoundStatementType is true for constructor declarations', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + โšExample() + { + } + } + `); + }); + + test('node.isCompoundStatementType is true for destructor declarations', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + โš~Example() + { + } + } + `); + }); + + test('node.isCompoundStatementType is true for blocks', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โš{ + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for checked statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšchecked + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for do statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšdo + { + } while (false); + } + } + `); + }); + + test('node.isCompoundStatementType is true for fixed statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšfixed + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for for statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšfor (;;) + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for lock statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšlock (x) + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for switch statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšswitch (x) + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for try statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โštry + { + } + finally + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for unsafe statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšunsafe + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for foreach statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšforeach (var item in items) + { + } + } + } + `); + }); + + test('node.isCompoundStatementType is true for uncollapsed if statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšif (x) + { + } + } + } + `); + + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšif (x) {} + } + } + `); + }); + + test('node.isCompoundStatementType is false for collapsed if statements', async function () { + await assertStatementIsNotCompoundType(dedent` + class Example + { + void Run() + { + โšif (x) return; + } + } + `); + }); + + test('node.isCompoundStatementType is true for while statements', async function () { + await assertStatementIsCompoundType(dedent` + class Example + { + void Run() + { + โšwhile (false) + { + } + } + } + `); + }); + + async function assertStatementIsCompoundType(text: string) { + await testStatementIsCompoundType('csharp', text, true); + } + + async function assertStatementIsNotCompoundType(text: string) { + await testStatementIsCompoundType('csharp', text, false); + } + }); + + // MARK: C, C++ + + suite('C, C++', function () { + const languages = ['c', 'cpp']; + languages.forEach(lang => { + test(`${lang} is supported`, function () { + assert.strictEqual(StatementTree.isSupported(lang), true); + }); + }); + + suite('Statement identification (C, C++)', function () { + test('recognizes extern declarations', async function () { + await testStatementBuilding('c', `โ–ถ๏ธextern int foo();โ—€๏ธ`); + }); + + test('recognizes typedef declarations', async function () { + await testStatementBuilding('c', `โ–ถ๏ธtypedef int myInt;โ—€๏ธ`); + }); + + test('recognizes struct declarations', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธtypedef struct Obj + โ–ถ๏ธ{ + โ–ถ๏ธint x;โ—€๏ธ + โ–ถ๏ธfloat y;โ—€๏ธ + }โ—€๏ธ obj;โ—€๏ธ + ` + ); + }); + + test('recognizes union declarations', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธunion Example + โ–ถ๏ธ{ + โ–ถ๏ธint x;โ—€๏ธ + โ–ถ๏ธfloat y;โ—€๏ธ + }โ—€๏ธ exampleโ—€๏ธ + ` + ); + }); + + test('recognizes enum declarations', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธenum Color + { + RED, + GREEN, + BLUE + }โ—€๏ธ + ` + ); + }); + + test('recognizes function declarations', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธint add(int a, int b) + โ–ถ๏ธ{ + โ–ถ๏ธreturn a + b;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes old style function declarations', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธint add(a, b)โ—€๏ธ + โ–ถ๏ธint a;โ—€๏ธ + โ–ถ๏ธint b;โ—€๏ธ + โ–ถ๏ธ{ + โ–ถ๏ธreturn a + b;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes variable declarations', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธint x = 10;โ—€๏ธ + ` + ); + }); + + test('recognizes compound statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธ{ + โ–ถ๏ธint x = 10;โ—€๏ธ + โ–ถ๏ธint y = 20;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes if statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธvoid example() + โ–ถ๏ธ{ + โ–ถ๏ธif (x > 0) + โ–ถ๏ธ{ + โ–ถ๏ธprintf("Positive");โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes else and else if statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธvoid example() + โ–ถ๏ธ{ + โ–ถ๏ธif (x > 0) + โ–ถ๏ธ{ + โ–ถ๏ธprintf("Positive");โ—€๏ธ + }โ—€๏ธelse โ–ถ๏ธif (x < 0) + โ–ถ๏ธ{ + โ–ถ๏ธprintf("Negative");โ—€๏ธ + }โ—€๏ธ + else + โ–ถ๏ธ{ + โ–ถ๏ธprintf("Zero");โ—€๏ธ + }โ—€๏ธโ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes switch statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธvoid example() + โ–ถ๏ธ{ + โ–ถ๏ธswitch (x) + โ–ถ๏ธ{ + โ–ถ๏ธcase 1: + โ–ถ๏ธprintf("One");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธโ—€๏ธ + โ–ถ๏ธcase 2: + โ–ถ๏ธprintf("Two");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธโ—€๏ธ + โ–ถ๏ธdefault: + โ–ถ๏ธprintf("Default");โ—€๏ธ + โ–ถ๏ธbreak;โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes while statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธvoid example() + โ–ถ๏ธ{ + โ–ถ๏ธwhile (x < 10) + โ–ถ๏ธ{ + โ–ถ๏ธprintf("%d", x);โ—€๏ธ + โ–ถ๏ธx++;โ—€๏ธ + โ–ถ๏ธcontinue;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes for statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธvoid example() + โ–ถ๏ธ{ + โ–ถ๏ธfor (โ–ถ๏ธint i = 0;โ—€๏ธ i < 10; i++) + โ–ถ๏ธ{ + โ–ถ๏ธprintf("%d", i);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes do while statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธvoid example() + โ–ถ๏ธ{ + โ–ถ๏ธdo + โ–ถ๏ธ{ + โ–ถ๏ธprintf("%d", x);โ—€๏ธ + }โ—€๏ธ while (x < 10);โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes goto statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธgoto label;โ—€๏ธ + โ–ถ๏ธlabel: + โ–ถ๏ธprintf("Label reached");โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes preprocessor if statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธ#if DEBUG + โ–ถ๏ธ#define STACK 0 + โ—€๏ธ#elif RELEASE + โ–ถ๏ธ#define STACK 100 + โ—€๏ธ#else + โ–ถ๏ธprintf("Unknown mode");โ—€๏ธ + #endifโ—€๏ธ + ` + ); + }); + + test('recognizes ifdef statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธ#ifdef DEBUG + โ–ถ๏ธprintf("Debug mode");โ—€๏ธ + #endifโ—€๏ธ + ` + ); + }); + + test('recognizes include statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธ#include <stdio.h> + โ—€๏ธโ–ถ๏ธ#include "myheader.h"โ—€๏ธ + ` + ); + }); + + test('recognizes preprocessor call statements', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธ#import "..\\file" + โ—€๏ธโ–ถ๏ธ#line 10 + โ—€๏ธโ–ถ๏ธ#pragma once + โ—€๏ธโ–ถ๏ธ#using "using_assembly_A.dll" + โ—€๏ธโ–ถ๏ธ#undef ADD + โ—€๏ธโ–ถ๏ธ#error C++ compiler required.โ—€๏ธ + ` + ); + }); + + test('recognizes preprocessor functions', async function () { + await testStatementBuilding( + 'c', + dedent` + โ–ถ๏ธ#define SQUARE(x) ((x) * (x)) + โ—€๏ธโ–ถ๏ธ#define MAX(a, b) (\\ + (a) > (b) ? (a) : (b) \\ + )โ—€๏ธ + ` + ); + }); + }); + + suite('Statement identification (C++)', function () { + test('recognizes namespace statements', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธnamespace MyNamespace + { + โ–ถ๏ธint x;โ—€๏ธ + }โ—€๏ธ + ` + ); + }); + + test('recognizes class definitions', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธclass MyClass + โ–ถ๏ธ{ + โ–ถ๏ธint x;โ—€๏ธ + โ–ถ๏ธvoid m() โ–ถ๏ธ{ + โ–ถ๏ธx = 1;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes template declarations', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธtemplate <typename T> โ–ถ๏ธT myMax(T x, T y) โ–ถ๏ธ{ + โ–ถ๏ธreturn (x > y) ? x : y;โ—€๏ธ + }โ—€๏ธโ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes concept definitions', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธtemplate<typename T> + โ–ถ๏ธconcept MyConcept = requires(T t) + { + โ–ถ๏ธ{ t.foo() } -> std::same_as<int>;โ—€๏ธ + }โ—€๏ธโ—€๏ธ + ` + ); + }); + + test('recognizes using statements', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธusing MyType = int;โ—€๏ธ + ` + ); + }); + + test('recognizes alias declarations', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธusing MyAlias = int;โ—€๏ธ + ` + ); + }); + + test('recognizes static assertions', async function () { + await testStatementBuilding( + 'cpp', + dedent` + โ–ถ๏ธstatic_assert(sizeof(int) == 4, "int is not 4 bytes");โ—€๏ธ + ` + ); + }); + }); + + suite('Compound Statement Identification (C, C++)', function () { + test('node.isCompoundStatementType is true for struct declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšstruct Obj + { + int x; + float y; + } obj; + `); + }); + + test('node.isCompoundStatementType is true for union declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšunion Obj + { + int x; + float y; + } obj; + `); + }); + + test('node.isCompoundStatementType is true for enum declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšenum Color + { + RED, + GREEN, + BLUE + } obj; + `); + }); + + test('node.isCompoundStatementType is true for empty blocks', async function () { + await assertStatementIsCompoundType(dedent` + โš{ + } + `); + }); + + test('node.isCompoundStatementType is true for function declarations', async function () { + await assertStatementIsCompoundType(dedent` + void example() + { + โšint add(int a, int b) + { + return a + b; + } + } + `); + }); + + test('node.isCompoundStatementType is true for compound statements', async function () { + await assertStatementIsCompoundType(dedent` + โš{ + int x = 10; + int y = 20; + } + `); + }); + + test('node.isCompoundStatementType is true for if statements', async function () { + await assertStatementIsCompoundType(dedent` + void example() + { + โšif (x > 0) + { + printf("Positive"); + } + } + `); + }); + + test('node.isCompoundStatementType is true for type definitions', async function () { + await assertStatementIsCompoundType(dedent` + โštypedef struct Obj + { + int x; + float y; + } obj; + `); + }); + + test('node.isCompoundStatementType is true for for statements', async function () { + await assertStatementIsCompoundType(dedent` + void example() + { + โšfor (int i = 0; i < 10; i++) + { + printf("%d", i); + } + } + `); + }); + + test('node.isCompoundStatementType is true for while statements', async function () { + await assertStatementIsCompoundType(dedent` + void example() + { + โšwhile (x < 10) + { + printf("%d", x); + x++; + } + } + `); + }); + + test('node.isCompoundStatementType is true for do while statements', async function () { + await assertStatementIsCompoundType(dedent` + void example() + { + โšdo + { + printf("%d", x); + } while (x < 10); + } + `); + }); + + test('node.isCompoundStatementType is true for switch statements', async function () { + await assertStatementIsCompoundType(dedent` + void example() + { + โšswitch (x) + { + default: + printf("Default"); + break; + } + } + `); + }); + + test('node.isCompoundStatementType is true for preprocessor if statements', async function () { + await assertStatementIsCompoundType(dedent` + โš#if DEBUG + #define STACK 0 + #elif RELEASE + #define STACK 100 + #else + printf("Unknown mode"); + #endif + `); + }); + + test('node.isCompoundStatementType is true for preprocessor ifdef statements', async function () { + await assertStatementIsCompoundType(dedent` + โš#ifdef DEBUG + printf("Debug mode"); + #endif + `); + }); + + test('node.isCompoundStatementType is false for declaration statements', async function () { + await assertStatementIsNotCompoundType(dedent` + int foo() { + โšint x = 10; + } + `); + }); + + test('node.isCompoundStatementType is false for return statements', async function () { + await assertStatementIsNotCompoundType(dedent` + int foo() { + โšreturn 1; + } + `); + }); + + test('node.isCompoundStatementType is false for goto statements', async function () { + await assertStatementIsNotCompoundType(dedent` + โšgoto label; + `); + }); + + test('node.isCompoundStatementType is false for label statements', async function () { + await assertStatementIsNotCompoundType(dedent` + โšlabel: + printf("Label reached"); + `); + }); + + test('node.isCompoundStatementType is false for preprocessor include statements', async function () { + await assertStatementIsNotCompoundType(dedent` + โš#include <stdio.h> + `); + }); + + test('node.isCompoundStatementType is false for preprocessor functions', async function () { + await assertStatementIsNotCompoundType(dedent` + โš#define SQUARE(x) ((x) * (x)) + `); + }); + + async function assertStatementIsCompoundType(text: string) { + await testStatementIsCompoundType('c', text, true); + } + + async function assertStatementIsNotCompoundType(text: string) { + await testStatementIsCompoundType('c', text, false); + } + }); + + suite('Compound Statement Identification (C++)', function () { + test('node.isCompoundStatementType is true for namespace definitions', async function () { + await assertStatementIsCompoundType(dedent` + โšnamespace MyNamespace + { + int x; + } + `); + }); + + test('node.isCompoundStatementType is true for template declaratations', async function () { + await assertStatementIsCompoundType(dedent` + โštemplate<typename T> + class MyClass + { + T value; + } + `); + }); + + test('node.isCompoundStatementType is true for concept definitions', async function () { + await assertStatementIsCompoundType(dedent` + โšconcept MyConcept = requires(T t) + { + { t.foo() } -> std::same_as<int>; + }; + `); + }); + + test('node.isCompoundStatementType is true for class declarations', async function () { + await assertStatementIsCompoundType(dedent` + โšclass MyClass + { + int x; + float y; + }; + `); + }); + + test('node.isCompoundStatementType is true for class declarations with template', async function () { + await assertStatementIsCompoundType(dedent` + โštemplate<typename T> + class MyClass + { + T value; + }; + `); + }); + + test('node.isCompoundStatementType is true for field declaration lists', async function () { + await assertStatementIsCompoundType(dedent` + class MyClass + โš{ + int x; + float y; + double z; + }; + `); + }); + + test('node.isCompoundStatementType is false for field declarations', async function () { + await assertStatementIsNotCompoundType(dedent` + class MyClass + { + โšint x; + float y; + }; + `); + }); + + test('node.isCompoundStatementType is false for single-line concept definitions', async function () { + await assertStatementIsNotCompoundType(dedent` + template<class T, class U> + โšconcept Derived = std::is_base_of<U, T>::value; + `); + }); + + test('node.isCompoundStatementType is false for using statements', async function () { + await assertStatementIsNotCompoundType(dedent` + โšusing MyType = int; + `); + }); + + test('node.isCompoundStatementType is false for alias declarations', async function () { + await assertStatementIsNotCompoundType(dedent` + โšusing MyAlias = int; + `); + }); + + test('node.isCompoundStatementType is false for static assertions', async function () { + await assertStatementIsNotCompoundType(dedent` + โšstatic_assert(sizeof(int) == 4, "int is not 4 bytes"); + `); + }); + + async function assertStatementIsCompoundType(text: string) { + await testStatementIsCompoundType('cpp', text, true); + } + + async function assertStatementIsNotCompoundType(text: string) { + await testStatementIsCompoundType('cpp', text, false); + } + }); + }); + + /** + * Use `โ–ถ๏ธ` and `โ—€๏ธ` to mark the beginning and end of statements in the test text. + * + * If `โš` (`'\u275A'`) is present in the text, it represents the cursor, and the region + * between the cursor and end of the text is passed as the offsets for tree building + * (otherwise, the full text region is used). + */ + async function testStatementBuilding(language: string, text: string) { + const delim = /โ–ถ๏ธ|โ—€๏ธ|โš/; + const statements: StatementNodeSpec[] = []; + let doc = ''; + let remainder = text; + let s: StatementNodeSpec | undefined; + let match = remainder.match(delim); + let startOffset = 0; + + while (match) { + doc += remainder.slice(0, match.index); + if (match[0] === 'โ–ถ๏ธ') { + const newS: StatementNodeSpec = { + startOffset: doc.length, + parent: s, + children: [], + }; + if (s) { + s.children.push(newS); + } else { + statements.push(newS); + } + s = newS; + } else if (match[0] === 'โš') { + startOffset = doc.length; + } else { + if (s) { + s.endOffset = doc.length; + s = s.parent; + } else { + throw new Error( + `Unmatched statement end at offset ${doc.length} (at ${JSON.stringify(remainder.slice(match.index! + match[0].length))})` + ); + } + } + remainder = remainder.slice(match.index! + match[0].length); + match = remainder.match(delim); + } + doc += remainder; + + if (s) { + throw new Error( + `Unmatched statement start beginning at offset ${s.startOffset} (at ${JSON.stringify(doc.substring(s.startOffset))})` + ); + } + + using tree = StatementTree.create(language, doc, startOffset, doc.length); + + await tree.build(); + + function expectNodeLike(node: StatementNode, spec: StatementNodeSpec, prefix = '') { + const pad = ' '.repeat(prefix.length); + const path = node.dumpPath(prefix, pad); + assert.strictEqual( + node.node.startIndex, + spec.startOffset, + `At:\n\n${path}\n\nExpected statement to begin at offset ${spec.startOffset}, but begins at ${node.node.startIndex}` + ); + assert.strictEqual( + node.node.endIndex, + spec.endOffset, + `At:\n\n${path}\n\nExpected statement to end at offset ${spec.endOffset}, but ends at ${node.node.endIndex}` + ); + assert.strictEqual( + node.children.length, + spec.children.length, + `At:\n\n${path}\n\nExpected node to have ${spec.children.length} children, but got ${node.children.length}` + ); + for (let i = 0; i < spec.children.length; i++) { + expectNodeLike(node.children[i], spec.children[i], prefix); + } + } + + assert.strictEqual( + tree.statements.length, + statements.length, + `Expected a tree with ${statements.length} statements, but got ${tree.statements.length}:\n${tree.dump()}` + ); + for (let i = 0; i < statements.length; i++) { + expectNodeLike(tree.statements[i], statements[i], ` [${i}] `); + } + } + + async function testStatementIsCompoundType(languageId: string, text: string, expectedResult: boolean) { + const posIndicator = 'โš'; + const offset = text.indexOf(posIndicator); + const doc = text.replace(posIndicator, ''); + using tree = StatementTree.create(languageId, doc, 0, doc.length); + + await tree.build(); + const statement = tree.statementAt(offset + 1); + + assert.ok(statement, `Statement not found at offset ${offset}`); + assert.strictEqual( + statement.isCompoundStatementType, + expectedResult, + `Expected .isCompoundStatementType to be ${expectedResult ? 'true' : 'false'} for ${statement.node.type} but got ${statement.isCompoundStatementType ? 'true' : 'false'}` + ); + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/ghostText/test/streamedCompletionSplitter.test.ts b/completions-sample-code/vscode-node/lib/src/ghostText/test/streamedCompletionSplitter.test.ts new file mode 100644 index 0000000..5528f5a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/ghostText/test/streamedCompletionSplitter.test.ts @@ -0,0 +1,297 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import assert from 'assert'; +import Sinon from 'sinon'; +import dedent from 'ts-dedent'; +import { SyncDescriptor } from '../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFetcherService } from '../../networking'; +import { CompletionResults, CopilotUiKind, ICompletionsOpenAIFetcherService, LiveOpenAIFetcher } from '../../openai/fetch'; +import { APIChoice } from '../../openai/openai'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { createFakeCompletionResponse, fakeCodeReference, StaticFetcher } from '../../test/fetcher'; +import { StreamedCompletionSplitter } from '../streamedCompletionSplitter'; + +suite('StreamedCompletionSplitter', function () { + function setupSplitter(fetcher: ICompletionsFetcherService, docPrefix = 'function example(arg) {\n', languageId = 'javascript') { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsFetcherService, fetcher); + serviceCollection.define(ICompletionsOpenAIFetcherService, new SyncDescriptor(LiveOpenAIFetcher)); // gets results from static fetcher + const accessor = serviceCollection.createTestingAccessor(); + + const fetcherService = accessor.get(ICompletionsOpenAIFetcherService); + const telemetry = TelemetryWithExp.createEmptyConfigForTesting(); + const params = { + prompt: { + prefix: docPrefix, + suffix: '', + isFimEnabled: false, + promptElementRanges: [], + }, + languageId: languageId, + repoInfo: undefined, + ourRequestId: 'test-request-id', + engineModelId: 'test-model-id', + count: 1, + uiKind: CopilotUiKind.GhostText, + extra: {}, + }; + const cacheFunction = Sinon.stub<[string, APIChoice], void>(); + const splitter = accessor.get(IInstantiationService).createInstance(StreamedCompletionSplitter, docPrefix, languageId, true, 7, cacheFunction); + const fetchAndStreamCompletions = async function () { + return await fetcherService.fetchAndStreamCompletions(params, telemetry, splitter.getFinishedCallback()); + }; + return { splitter, cacheFunction, fetchAndStreamCompletions }; + } + + async function readChoices(result: CompletionResults): Promise<APIChoice[]> { + const choices = []; + for await (const choice of result.choices) { + choices.push(choice); + } + return choices; + } + + test('yields the first line of the completion', async function () { + const { fetchAndStreamCompletions } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse( + dedent` + const result = []; + for (let i = 0; i < arg; i++) { + result.push(i); + } + return result.join(', '); + ` + ) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + const completions = await readChoices(result); + assert.strictEqual(completions.length, 1); + assert.strictEqual(completions[0].completionText, 'const result = [];'); + }); + + test('caches the remaining sections of the completion', async function () { + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse( + dedent` + const result = []; + for (let i = 0; i < arg; i++) { + result.push(i); + } + return result.join(', '); + ` + ) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledTwice(cacheFunction); + Sinon.assert.calledWith( + cacheFunction, + 'const result = [];', + Sinon.match({ + completionText: '\nfor (let i = 0; i < arg; i++) {\n\tresult.push(i);\n}', + }) + ); + Sinon.assert.calledWith( + cacheFunction, + 'const result = [];\nfor (let i = 0; i < arg; i++) {\n\tresult.push(i);\n}', + Sinon.match({ completionText: `\nreturn result.join(', ');` }) + ); + }); + + test('trims trailing whitespace from cached completions', async function () { + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => createFakeCompletionResponse('// one\n\n// two ')) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledWith(cacheFunction, '// one', Sinon.match({ completionText: '\n\n// two' })); + }); + + test('allows single line completions that begin with a newline', async function () { + const { fetchAndStreamCompletions } = setupSplitter( + new StaticFetcher(() => createFakeCompletionResponse('\n// one\n// two')) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + const completions = await readChoices(result); + assert.strictEqual(completions.length, 1); + assert.strictEqual(completions[0].completionText, '\n// one'); + }); + + test('allows single line completions that begin with a CRLF pair', async function () { + const { fetchAndStreamCompletions } = setupSplitter( + new StaticFetcher(() => createFakeCompletionResponse('\r\n// one\r\n// two')) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + const completions = await readChoices(result); + assert.strictEqual(completions.length, 1); + assert.strictEqual(completions[0].completionText, '\r\n// one'); + }); + + test('sets generatedChoiceIndex on cached completions', async function () { + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse( + dedent` + const result = []; + for (let i = 0; i < arg; i++) { + result.push(i); + } + return result.join(', '); + ` + ) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledWith(cacheFunction, Sinon.match.string, Sinon.match({ generatedChoiceIndex: 1 })); + Sinon.assert.calledWith(cacheFunction, Sinon.match.string, Sinon.match({ generatedChoiceIndex: 2 })); + }); + + test('adjusts start_offset in any annotations present in cached split choices', async function () { + const parts = ['x=1;', '\n\ny=2;', '\n\nz=3;\n']; + const completion = parts.join(''); + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse(completion, { annotations: fakeCodeReference(-1, completion.length + 1) }) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledTwice(cacheFunction); + Sinon.assert.calledWith( + cacheFunction, + Sinon.match.string, + Sinon.match({ + copilotAnnotations: Sinon.match({ + ip_code_citations: [Sinon.match({ start_offset: -parts[0].length - 1 })], + }), + }) + ); + Sinon.assert.calledWith( + cacheFunction, + Sinon.match.string, + Sinon.match({ + copilotAnnotations: Sinon.match({ + ip_code_citations: [Sinon.match({ start_offset: -parts[0].length - parts[1].length - 1 })], + }), + }) + ); + }); + + test('adjusts stop_offset in any annotations present in cached split choices', async function () { + const parts = ['x=1;', '\n\ny=2;', '\n\nz=3;']; + const completion = parts.join(''); + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse(completion, { annotations: fakeCodeReference(-1, completion.length + 1) }) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledTwice(cacheFunction); + Sinon.assert.calledWith( + cacheFunction, + Sinon.match.string, + Sinon.match({ + copilotAnnotations: Sinon.match({ + ip_code_citations: [Sinon.match({ stop_offset: parts[1].length })], + }), + }) + ); + Sinon.assert.calledWith( + cacheFunction, + Sinon.match.string, + Sinon.match({ + copilotAnnotations: Sinon.match({ + ip_code_citations: [Sinon.match({ stop_offset: parts[2].length + 1 })], + }), + }) + ); + }); + + test('omits any annotation from split choices where start_offset does not intersect the choice', async function () { + const parts = ['x=1;', '\n\ny=2;', '\n\nz=3;\n']; + const completion = parts.join(''); + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse(completion, { + annotations: fakeCodeReference(parts[0].length + parts[1].length + 3, completion.length + 1), + }) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledTwice(cacheFunction); + Sinon.assert.calledWith(cacheFunction, Sinon.match.string, Sinon.match({ copilotAnnotations: undefined })); + Sinon.assert.calledWith( + cacheFunction, + Sinon.match.string, + Sinon.match({ + copilotAnnotations: Sinon.match({ + ip_code_citations: [Sinon.match({ start_offset: 3 })], + }), + }) + ); + }); + + test('omits any annotation from split choices where stop_offset does not intersect the choice', async function () { + const parts = ['x=1;', '\n\ny=2;', '\n\nz=3;\n']; + const completion = parts.join(''); + const { fetchAndStreamCompletions, cacheFunction } = setupSplitter( + new StaticFetcher(() => + createFakeCompletionResponse(completion, { annotations: fakeCodeReference(-1, parts[0].length + 3) }) + ) + ); + + const result = await fetchAndStreamCompletions(); + + assert.strictEqual(result.type, 'success'); + await readChoices(result); + Sinon.assert.calledTwice(cacheFunction); + Sinon.assert.calledWith( + cacheFunction, + Sinon.match.string, + Sinon.match({ + copilotAnnotations: Sinon.match({ + ip_code_citations: [Sinon.match({ stop_offset: 3 })], + }), + }) + ); + Sinon.assert.calledWith(cacheFunction, Sinon.match.string, Sinon.match({ copilotAnnotations: undefined })); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/helpers/cache.ts b/completions-sample-code/vscode-node/lib/src/helpers/cache.ts new file mode 100644 index 0000000..04e62f0 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/helpers/cache.ts @@ -0,0 +1,99 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + + +/** + * This implements the Map interface. Note that in all methods that iterate or return an iterator, a copy of the underlying data is + * returned so that if you call `get`, `set`, or `delete` while iterating, the iterator will not be invalidated. + */ +export class LRUCacheMap<K, T> implements Map<K, T> { + private valueMap = new Map<K, T>(); + private sizeLimit: number; + + // constructor + constructor(size = 10) { + if (size < 1) { + throw new Error('Size limit must be at least 1'); + } + this.sizeLimit = size; + } + + set(key: K, value: T): this { + if (this.has(key)) { + // If key already exists, delete it + // from the valueMap only so we can re-insert it at the end + this.valueMap.delete(key); + } else if (this.valueMap.size >= this.sizeLimit) { + // least-recently used cache eviction strategy + // Maps iterate in insertion order + const oldest = this.valueMap.keys().next().value!; + this.delete(oldest); + } + + this.valueMap.set(key, value); + return this; + } + + /** + * Warning this method makes the key the most recently used. To avoid this, use `peek` instead. + * @param key + * @returns + */ + get(key: K): T | undefined { + if (this.valueMap.has(key)) { + const entry = this.valueMap.get(key); + // Move to the end by deleting and re-inserting + this.valueMap.delete(key); + this.valueMap.set(key, entry!); + return entry!; + } + + return undefined; + } + + delete(key: K): boolean { + return this.valueMap.delete(key); + } + + clear() { + this.valueMap.clear(); + } + + get size(): number { + return this.valueMap.size; + } + + keys(): IterableIterator<K> { + return new Map(this.valueMap).keys(); + } + + values(): IterableIterator<T> { + return new Map(this.valueMap).values(); + } + + entries(): IterableIterator<[K, T]> { + return new Map(this.valueMap).entries(); + } + + [Symbol.iterator](): IterableIterator<[K, T]> { + return this.entries(); + } + + has(key: K): boolean { + return this.valueMap.has(key); + } + + forEach(callbackfn: (value: T, key: K, map: Map<K, T>) => void, thisArg?: unknown): void { + new Map(this.valueMap).forEach(callbackfn, thisArg); + } + + get [Symbol.toStringTag](): string { + return 'LRUCacheMap'; + } + + peek(key: K): T | undefined { + return this.valueMap.get(key); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/helpers/iterableHelpers.ts b/completions-sample-code/vscode-node/lib/src/helpers/iterableHelpers.ts new file mode 100644 index 0000000..2578f57 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/helpers/iterableHelpers.ts @@ -0,0 +1,85 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export async function* asyncIterableMap<TSource, TDest>( + source: AsyncIterable<TSource>, + selector: (x: TSource) => Promise<TDest> | TDest +): AsyncIterable<TDest> { + for await (const item of source) { + yield selector(item); + } +} + +export async function* asyncIterableFilter<TSource>( + source: AsyncIterable<TSource>, + predicate: (x: TSource) => Promise<boolean> | boolean +): AsyncIterable<TSource> { + for await (const item of source) { + if (await predicate(item)) { + yield item; + } + } +} + +export async function* asyncIterableMapFilter<TSource, TDest>( + source: AsyncIterable<TSource>, + selector: (x: TSource) => Promise<TDest | undefined> | TDest | undefined +): AsyncIterable<TDest> { + for await (const item of source) { + const result = await selector(item); + if (result !== undefined) { + yield result; + } + } +} + +export async function* asyncIterableFromArray<TSource>(source: TSource[]): AsyncIterable<TSource, void, unknown> { + for (const item of source) { + yield Promise.resolve(item); + } +} + +export async function asyncIterableToArray<TSource>(source: AsyncIterable<TSource>): Promise<TSource[]> { + const result: TSource[] = []; + for await (const item of source) { + result.push(item); + } + return result; +} + +export async function* asyncIterableConcat<TSource>(...sources: AsyncIterable<TSource>[]): AsyncIterable<TSource> { + for (const source of sources) { + yield* source; + } +} + +export async function asyncIterableCount<TSource>(source: AsyncIterable<TSource>): Promise<number> { + let count = 0; + for await (const _ of source) { + count++; + } + return count; +} + +export function* iterableMap<TSource, TDest>( + source: Iterable<TSource>, + selector: (x: TSource) => TDest +): Iterable<TDest> { + for (const item of source) { + yield selector(item); + } +} + +export function* iterableMapFilter<TSource, TDest>( + source: Iterable<TSource>, + selector: (x: TSource) => TDest | undefined +): Iterable<TDest> { + for (const item of source) { + const result = selector(item); + if (result !== undefined) { + yield result; + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/helpers/radix.ts b/completions-sample-code/vscode-node/lib/src/helpers/radix.ts new file mode 100644 index 0000000..e1eb8fe --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/helpers/radix.ts @@ -0,0 +1,232 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** A data structure for efficiently finding all values that are indexed by a key + * that is a prefix of a given key, using a radix trie representation. + * + * An overarching goal of the implementation is to minimize storing and handling + * the full keys since in the case of completions, the keys are the full text of + * the document before the cursor which can be large. + */ +export class LRURadixTrie<T> { + /** Singular, empty root node for the the trie. */ + private readonly root = new LRURadixNode<T>(); + + /** Set of all leaf nodes with values, tracked for evicting LRU values. */ + private readonly leafNodes: Set<LRURadixNode<T>> = new Set(); + + constructor(private readonly maxSize: number) { } + + /** + * Traverses the trie to insert a new value. If an existing exact match is + * found the value is added to a list of values at that node. Otherwise a + * new node is created. + * + * As a side effect, the least recently used node is evicted if the max size + * is exceeded. + */ + set(key: string, value: T): void { + let { node, remainingKey } = this.findClosestNode(key); + // If no exact match, add a new node under the closest node. + if (remainingKey.length > 0) { + // Check if there is a child node with an edge that is a prefix of + // the remaining key. + for (const [edge, child] of node.children) { + if (edge.startsWith(remainingKey)) { + // Split the edge by adding a new intermediate node. + const commonPrefix = edge.slice(0, remainingKey.length); + const intermediate = new LRURadixNode<T>(); + node.removeChild(edge); + node.addChild(commonPrefix, intermediate); + intermediate.addChild(edge.slice(commonPrefix.length), child); + node = intermediate; + remainingKey = remainingKey.slice(commonPrefix.length); + break; + } + } + if (remainingKey.length > 0) { + // Add a new node with the remaining key. + const newNode = new LRURadixNode<T>(); + node.addChild(remainingKey, newNode); + node = newNode; + } + } + // Set value on the node + node.value = value; + // Ensure the node which may be new or newly with a value is in the + // leafNode set. + this.leafNodes.add(node); + // Evict least recently used node if max size is exceeded. + if (this.leafNodes.size > this.maxSize) { + this.evictLeastRecentlyUsed(); + } + } + + /** Traverses the trie and returns all values whose keys are a prefix of the + * given key. Returns them in order of longest prefix first. + */ + findAll(key: string): Array<{ remainingKey: string; value: T }> { + return this.findClosestNode(key) + .stack.map(({ node, remainingKey }) => + node.value !== undefined ? { remainingKey, value: node.value } : undefined + ) + .filter(x => x !== undefined); + } + + /** Removes the value at a given key if any from the trie. */ + delete(key: string): void { + const { node, remainingKey } = this.findClosestNode(key); + // If no exact match is found, do nothing. + if (remainingKey.length > 0) { return; } + // Exact match found, remove the value. + this.deleteNode(node); + } + + /** Traverses the trie to find the node with the closest prefix to a given key. */ + private findClosestNode(key: string) { + let hasNext = true; + let node: LRURadixNode<T> = this.root; + const stack: { node: LRURadixNode<T>; remainingKey: string }[] = [{ node, remainingKey: key }]; + while (key.length > 0 && hasNext) { + hasNext = false; + for (const [edge, child] of node.children) { + if (key.startsWith(edge)) { + key = key.slice(edge.length); + stack.unshift({ node: child, remainingKey: key }); + node = child; + hasNext = true; + break; + } + } + } + return { node, remainingKey: key, stack }; + } + + /** Deletes a node from the trie and resolves relationships with surrounding nodes. + * - If the node has no children, remove it from its parent. + * - If the node has one child, replace it with its child in the parent, + * concatenating the edges together. + * - If the node has multiple children, the node is left in place as an + * intermediary node. + * - In all cases, the value at the node is removed and the node is removed + * from the flatNodes set of leaf nodes. + */ + private deleteNode(node: LRURadixNode<T>): void { + node.value = undefined; + this.leafNodes.delete(node); + // If the node has no parent, it is the root. Done. + if (node.parent === undefined) { return; } + // If more than one child, keep the node as an intermediary node. Done. + if (node.childCount > 1) { return; } + const { node: parent, edge } = node.parent; + // If exactly one child, replace the node with the child in the parent. + if (node.childCount === 1) { + const [childEdge, childNode] = Array.from(node.children)[0]; + node.removeChild(childEdge); + parent.removeChild(edge); + parent.addChild(edge + childEdge, childNode); + return; + } + // If the node has no children, remove it from the parent. + parent.removeChild(edge); + // If the parent node is the root, no further action is needed. + if (parent.parent === undefined) { return; } + const grandparent = parent.parent; + // If the parent node has only one child remaining and no value, merge + // the parent and remaining child together. + if (parent.value === undefined && parent.childCount === 1) { + const [childEdge, childNode] = Array.from(parent.children)[0]; + const newEdge = grandparent.edge + childEdge; + parent.removeChild(childEdge); + grandparent.node.removeChild(grandparent.edge); + grandparent.node.addChild(newEdge, childNode); + } + } + + /** Walks the trie to find and evict the least recently used node. This is + * intentionally optimized for read performance over write performance. + */ + private evictLeastRecentlyUsed(): void { + const node = this.findLeastRecentlyUsed(); + if (node) { this.deleteNode(node); } + } + + /** Iterate through the set of leaf nodes to find the least recently used. + * + * Note, this could be done more efficiently with a heap or even just + * keeping the list sorted. Currently, this is mirroring the LRUCacheMap + * implementation to optimize for read performance over write performance. + * Though this may be worth revisiting since both reading and writing are on + * the critical path for completions. + */ + private findLeastRecentlyUsed(): LRURadixNode<T> | undefined { + let least: LRURadixNode<T> | undefined; + for (const node of this.leafNodes) { + if (least === undefined || node.touched < least.touched) { + least = node; + } + } + return least; + } +} + +/** Internal node representation in a LRURadixTrie. + * - Optionally has a value to represent a leaf node. + * - Contains a list of child nodes, not mutually exclusive with having value. + * - If not a root, has a parent edge for traversal up the trie. + * - Maintains state on most recent access time for LRU eviction. + */ +class LRURadixNode<T> { + private readonly _children: Map<string, LRURadixNode<T>> = new Map(); + private _touched = performance.now(); + private _value: T | undefined; + + /** Reference to the parent node and edge to this node for backtracking. */ + parent: { node: LRURadixNode<T>; edge: string } | undefined; + + /** Iterator for the children of this node. */ + get children() { + return this._children.entries(); + } + + /** The number of children of this node. */ + get childCount() { + return this._children.size; + } + + /** Adds a child node to this node and sets its parent reference. */ + addChild(edge: string, child: LRURadixNode<T>): void { + this._children.set(edge, child); + child.parent = { node: this, edge }; + } + + /** Removes a child node from this node and clears its parent reference. */ + removeChild(edge: string): void { + const child = this._children.get(edge); + if (child) { child.parent = undefined; } + this._children.delete(edge); + } + + /** Reads the value and updates the touched timestamp. */ + get value(): T | undefined { + this.touch(); + return this._value; + } + + /** Sets value and updates the touched timestamp. */ + set value(value: T | undefined) { + this.touch(); + this._value = value; + } + + /** The last time (ms from process start) this node's value was accessed. */ + get touched(): number { + return this._touched; + } + + private touch(): void { + this._touched = performance.now(); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/helpers/test/cache.test.ts b/completions-sample-code/vscode-node/lib/src/helpers/test/cache.test.ts new file mode 100644 index 0000000..1046a39 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/helpers/test/cache.test.ts @@ -0,0 +1,137 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { LRUCacheMap } from '../cache'; +import * as assert from 'assert'; + +suite('LRUCacheMap', function () { + test('should add and retrieve entries using set and get methods', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + cache.set('c', 3); + assert.equal(cache.get('b'), 2); + assert.equal(cache.get('c'), 3); + assert.equal(cache.get('a'), undefined, 'a should have been removed from the cache'); + assert.equal(cache.size, 2); + }); + + test('should not increase size if the same object is added twice', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('a', 1); + assert.equal(cache.size, 1); + }); + + test('should maintain the order of the values consistent with the order that the items were added or retrieved', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + assert.equal(cache.get('a'), 1); // this should make 'b' the most recently used + assert.equal(cache.peek('b'), 2); // this should not change the order + assert.ok(cache.has('b')); // b should still be in the cache + cache.set('c', 3); + assert.deepEqual([...cache.keys()], ['a', 'c']); + assert.deepEqual([...cache.values()], [1, 3]); + assert.ok(!cache.has('b')); // b should have been removed from the cache + assert.equal(cache.get('b'), undefined, 'b should have been removed from the cache'); + assert.equal(cache.get('z'), undefined, 'z was never added to the cache'); + assert.equal(cache.size, 2); + }); + + test('should delete entries using the delete method and decrease size', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + cache.delete('a'); + assert.equal(cache.get('a'), undefined); + assert.equal(cache.size, 1); + }); + + test('clear works', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + cache.clear(); + assert.equal(cache.get('a'), undefined); + assert.equal(cache.get('b'), undefined); + assert.equal(cache.size, 0); + }); + + test('should iterate over all entries using a for...of loop', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + const entries: [string, number][] = []; + for (const [key, value] of cache) { + entries.push([key, value]); + // touch a should not change for loop contents even though it becomes most recently used in the LRU + cache.get('a'); + cache.set('c', 3); // similarly, adding a new entry should not change the for loop contents + } + assert.deepEqual(entries, [ + ['a', 1], + ['b', 2], + ]); + }); + + test('should iterate over all entries using the entries method', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + const entries: [string, number][] = []; + for (const [key, value] of cache.entries()) { + entries.push([key, value]); + // touch a should not change for loop contents even though it becomes most recently used in the LRU + cache.get('a'); + cache.set('c', 3); // similarly, adding a new entry should not change the for loop contents + } + assert.deepEqual(entries, [ + ['a', 1], + ['b', 2], + ]); + }); + + test('should iterate over all entries using the forEach method', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + const entries: [string, number][] = []; + cache.forEach((value, key) => { + entries.push([key, value]); + cache.clear(); // shouldn't affect contents of forEach loop + }); + assert.deepEqual(entries, [ + ['a', 1], + ['b', 2], + ]); + }); + + test('should iterate over all values using the values method', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + const values: number[] = []; + for (const value of cache.values()) { + values.push(value); + // touch a should not change for loop contents even though it becomes most recently used in the LRU + cache.get('a'); + cache.set('c', 3); // similarly, adding a new entry should not change the for loop contents + } + assert.deepEqual(values, [1, 2]); + }); + + test('should iterate over all keys using the keys method', function () { + const cache = new LRUCacheMap<string, number>(2); + cache.set('a', 1); + cache.set('b', 2); + const keys: string[] = []; + for (const key of cache.keys()) { + keys.push(key); + cache.clear(); // shouldn't affect contents of forEach loop + } + assert.deepEqual(keys, ['a', 'b']); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/helpers/test/iterableHelpers.test.ts b/completions-sample-code/vscode-node/lib/src/helpers/test/iterableHelpers.test.ts new file mode 100644 index 0000000..485bc5d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/helpers/test/iterableHelpers.test.ts @@ -0,0 +1,147 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { + asyncIterableConcat, + asyncIterableCount, + asyncIterableFilter, + asyncIterableFromArray, + asyncIterableMap, + asyncIterableMapFilter, + asyncIterableToArray, + iterableMap, + iterableMapFilter, +} from '../iterableHelpers'; + +class AsyncIterableTestHelper { + state = 0; // this is used to check that operations are suitably lazy + async *[Symbol.asyncIterator](): AsyncIterator<number> { + this.state = 1; + yield Promise.resolve(1); + this.state = 2; + yield Promise.resolve(2); + this.state = 3; + yield Promise.resolve(3); + this.state = 4; + } + constructor() { } +} + +suite('Async Iterable utilities', function () { + // Sanity check that the generator itself behaves as expected + test('generator', async function () { + const asyncIterableIn = new AsyncIterableTestHelper(); + const asyncIterable = asyncIterableIn; + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(asyncIterableIn.state, 0); + assert.deepStrictEqual(await asyncIterator.next(), { value: 1, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 1); + assert.deepStrictEqual(await asyncIterator.next(), { value: 2, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 2); + assert.deepStrictEqual(await asyncIterator.next(), { value: 3, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 3); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + assert.deepStrictEqual(asyncIterableIn.state, 4); + }); + + test('map', async function () { + const asyncIterableIn = new AsyncIterableTestHelper(); + const asyncIterable = asyncIterableMap(asyncIterableIn, v => Promise.resolve(v * 2)); + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(asyncIterableIn.state, 0); + assert.deepStrictEqual(await asyncIterator.next(), { value: 2, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 1); + assert.deepStrictEqual(await asyncIterator.next(), { value: 4, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 2); + assert.deepStrictEqual(await asyncIterator.next(), { value: 6, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 3); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + assert.deepStrictEqual(asyncIterableIn.state, 4); + }); + + test('filter', async function () { + const asyncIterableIn = new AsyncIterableTestHelper(); + const asyncIterable = asyncIterableFilter(asyncIterableIn, v => Promise.resolve(v % 2 === 0)); + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(asyncIterableIn.state, 0); + assert.deepStrictEqual(await asyncIterator.next(), { value: 2, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 2); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + assert.deepStrictEqual(asyncIterableIn.state, 4); + }); + + test('mapFilter', async function () { + const asyncIterableIn = new AsyncIterableTestHelper(); + const asyncIterable = asyncIterableMapFilter(asyncIterableIn, v => + Promise.resolve(v % 2 === 0 ? v / 2 : undefined) + ); + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(asyncIterableIn.state, 0); + assert.deepStrictEqual(await asyncIterator.next(), { value: 1, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 2); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + assert.deepStrictEqual(asyncIterableIn.state, 4); + }); + + test('mapFilter keeps non-undefined falsy values', async function () { + const asyncIterableIn = new AsyncIterableTestHelper(); + const asyncIterable = asyncIterableMapFilter(asyncIterableIn, v => Promise.resolve(v % 2 === 0 ? v / 2 : 0)); + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(asyncIterableIn.state, 0); + assert.deepStrictEqual(await asyncIterator.next(), { value: 0, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 1); + assert.deepStrictEqual(await asyncIterator.next(), { value: 1, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 2); + assert.deepStrictEqual(await asyncIterator.next(), { value: 0, done: false }); + assert.deepStrictEqual(asyncIterableIn.state, 3); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + assert.deepStrictEqual(asyncIterableIn.state, 4); + }); + + test('fromArray', async function () { + const asyncIterable = asyncIterableFromArray([1, 2]); + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(await asyncIterator.next(), { value: 1, done: false }); + assert.deepStrictEqual(await asyncIterator.next(), { value: 2, done: false }); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + }); + + test('toArray', async function () { + const expected = [1, 2, 3]; + const asyncIterable = asyncIterableFromArray(expected); + const actual = await asyncIterableToArray(asyncIterable); + assert.deepStrictEqual(actual, expected); + }); + + test('concat', async function () { + const asyncIterable1 = asyncIterableFromArray([1, 2]); + const asyncIterable2 = asyncIterableFromArray([3, 4]); + const asyncIterable = asyncIterableConcat(asyncIterable1, asyncIterable2); + const asyncIterator = asyncIterable[Symbol.asyncIterator](); + assert.deepStrictEqual(await asyncIterator.next(), { value: 1, done: false }); + assert.deepStrictEqual(await asyncIterator.next(), { value: 2, done: false }); + assert.deepStrictEqual(await asyncIterator.next(), { value: 3, done: false }); + assert.deepStrictEqual(await asyncIterator.next(), { value: 4, done: false }); + assert.deepStrictEqual(await asyncIterator.next(), { value: undefined, done: true }); + }); + + test('count', async function () { + const asyncIterable = asyncIterableFromArray([1, 2]); + assert.deepStrictEqual(await asyncIterableCount(asyncIterable), 2); + }); + + test('iterableMap', function () { + const source = [1, 2, 3][Symbol.iterator](); + const actual = iterableMap(source, v => v * 2); + assert.deepStrictEqual(Array.from(actual), [2, 4, 6]); + }); + + test('iterableMapFilter', function () { + const source = [1, 2, 3][Symbol.iterator](); + const actual = iterableMapFilter(source, v => (v % 2 !== 0 ? v * 2 : undefined)); + assert.deepStrictEqual(Array.from(actual), [2, 6]); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/helpers/test/radix.test.ts b/completions-sample-code/vscode-node/lib/src/helpers/test/radix.test.ts new file mode 100644 index 0000000..7be74bd --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/helpers/test/radix.test.ts @@ -0,0 +1,164 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + + +import { LRURadixTrie } from '../radix'; +import * as assert from 'assert'; + +suite('LRURadixTrie', function () { + let trie: LRURadixTrie<string>; + + setup(function () { + trie = new LRURadixTrie<string>(20); + }); + + suite('set', function () { + test('stores a single value', function () { + trie.set('test', 'value'); + assert.deepStrictEqual(trie.findAll('test'), [{ remainingKey: '', value: 'value' }]); + }); + + test('splits edges when inserting', function () { + trie.set('test', 'first'); + trie.set('testing', 'second'); + assert.deepStrictEqual(trie.findAll('testing'), [ + { remainingKey: '', value: 'second' }, + { remainingKey: 'ing', value: 'first' }, + ]); + }); + + test('evicts least recently used when exceeding max size', function () { + trie = new LRURadixTrie<string>(3); + trie.set('a', 'first'); + trie.set('b', 'second'); + trie.set('c', 'third'); + trie.set('d', 'fourth'); + + assert.deepStrictEqual(trie.findAll('a'), []); + assert.deepStrictEqual(trie.findAll('b'), [{ remainingKey: '', value: 'second' }]); + assert.deepStrictEqual(trie.findAll('c'), [{ remainingKey: '', value: 'third' }]); + assert.deepStrictEqual(trie.findAll('d'), [{ remainingKey: '', value: 'fourth' }]); + }); + + test('shorter key as prefix of longer key', function () { + const trie = new LRURadixTrie<string>(20); + trie.set('test', '1'); + trie.set('t', '2'); + assert.deepStrictEqual(trie.findAll('test'), [ + { remainingKey: '', value: '1' }, + { remainingKey: 'est', value: '2' }, + ]); + }); + + test('insertion order does not matter', function () { + const trie1 = new LRURadixTrie<string>(20); + const trie2 = new LRURadixTrie<string>(20); + trie1.set('t', '2'); + trie1.set('test', '1'); + trie2.set('test', '1'); + trie2.set('t', '2'); + assert.deepStrictEqual(trie1.findAll('test'), [ + { remainingKey: '', value: '1' }, + { remainingKey: 'est', value: '2' }, + ]); + assert.deepStrictEqual(trie2.findAll('test'), [ + { remainingKey: '', value: '1' }, + { remainingKey: 'est', value: '2' }, + ]); + assert.deepStrictEqual(trie1.findAll('test'), trie2.findAll('test')); + }); + }); + + suite('findAll', function () { + test('returns all matching prefixes', function () { + trie.set('t', 'first'); + trie.set('te', 'second'); + trie.set('test', 'third'); + trie.set('test2', 'not expected'); + trie.set('team', 'not expected'); + trie.set('the', 'not expected'); + + assert.deepStrictEqual(trie.findAll('test'), [ + { remainingKey: '', value: 'third' }, + { remainingKey: 'st', value: 'second' }, + { remainingKey: 'est', value: 'first' }, + ]); + }); + + test('returns empty array when no matches found', function () { + trie.set('abc', 'value'); + trie.set('xyz1', 'value'); + trie.set('xyz2', 'value'); + assert.deepStrictEqual(trie.findAll('xyz'), []); + }); + + test('updates the least recently used when accessed', function () { + trie = new LRURadixTrie<string>(3); + trie.set('a', 'first'); + trie.set('b', 'second'); + trie.set('c', 'third'); + trie.findAll('a'); + trie.set('d', 'fourth'); + assert.deepStrictEqual(trie.findAll('b'), []); + assert.deepStrictEqual(trie.findAll('c'), [{ remainingKey: '', value: 'third' }]); + assert.deepStrictEqual(trie.findAll('d'), [{ remainingKey: '', value: 'fourth' }]); + assert.deepStrictEqual(trie.findAll('a'), [{ remainingKey: '', value: 'first' }]); + }); + }); + + suite('delete', function () { + test('removes a value', function () { + trie.set('test', 'value'); + trie.delete('test'); + assert.deepStrictEqual(trie.findAll('test'), []); + }); + + test('handles merging child node after delete', function () { + trie.set('test', 'first'); + trie.set('testing', 'second'); + + trie.delete('test'); + + assert.deepStrictEqual(trie.findAll('test'), []); + assert.deepStrictEqual(trie.findAll('testing'), [{ remainingKey: '', value: 'second' }]); + }); + + test('handles merging sibling node after delete', function () { + trie.set('test', 'first'); + trie.set('testing', 'second'); + trie.set('testy', 'third'); + + trie.delete('test'); + trie.delete('testing'); + + assert.deepStrictEqual(trie.findAll('test'), []); + assert.deepStrictEqual(trie.findAll('testing'), []); + assert.deepStrictEqual(trie.findAll('testy'), [{ remainingKey: '', value: 'third' }]); + }); + + test('does nothing when key not found', function () { + trie.set('test', 'value'); + trie.delete('other'); + assert.deepStrictEqual(trie.findAll('test'), [{ remainingKey: '', value: 'value' }]); + }); + }); + + test('handles unicode characters with multiple code points', function () { + /* Note: this behavior is arguably incorrect. Ideally, unicode characters + * comprising multiple code points would be treated as single characters. + * However, to do so would require converting all strings to arrays with + * Array.from and no longer using native string methods such as + * startsWith. This performance hit from that is likely not worth fixing + * this behavior. */ + trie.set('รฐลธยคยฆ', 'no modifiers'); + trie.set('รฐลธยคยฆรฐลธยยฝ', 'type 3'); + trie.set('รฐลธยคยฆรฐลธยยฝรขโ‚ฌยรขโ„ขโ€š', 'man type 3'); + assert.deepStrictEqual(trie.findAll('รฐลธยคยฆรฐลธยยฝรขโ‚ฌยรขโ„ขโ€šรฏยธย'), [ + { remainingKey: 'รฏยธย', value: 'man type 3' }, + { remainingKey: 'รขโ‚ฌยรขโ„ขโ€šรฏยธย', value: 'type 3' }, + { remainingKey: 'รฐลธยยฝรขโ‚ฌยรขโ„ขโ€šรฏยธย', value: 'no modifiers' }, + ]); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/inlineCompletion.ts b/completions-sample-code/vscode-node/lib/src/inlineCompletion.ts new file mode 100644 index 0000000..3a0e7ed --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/inlineCompletion.ts @@ -0,0 +1,125 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CancellationToken, Position, Range } from 'vscode-languageserver-protocol'; +import { IInstantiationService } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CompletionState, createCompletionState } from './completionState'; +import { completionsFromGhostTextResults, CopilotCompletion } from './ghostText/copilotCompletion'; +import { getGhostText, GetGhostTextOptions, ResultType } from './ghostText/ghostText'; +import { setLastShown } from './ghostText/last'; +import { ITextEditorOptions } from './ghostText/normalizeIndent'; +import { ICompletionsSpeculativeRequestCache } from './ghostText/speculativeRequestCache'; +import { GhostTextResultWithTelemetry, handleGhostTextResultTelemetry, logger } from './ghostText/telemetry'; +import { ICompletionsLogTargetService } from './logger'; +import { ITextDocument, TextDocumentContents } from './textDocument'; + +type GetInlineCompletionsOptions = Partial<GetGhostTextOptions> & { + formattingOptions?: ITextEditorOptions; +}; + +export class GhostText { + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsLogTargetService private readonly logTargetService: ICompletionsLogTargetService, + @ICompletionsSpeculativeRequestCache private readonly speculativeRequestCache: ICompletionsSpeculativeRequestCache, + ) { } + + public async getInlineCompletions( + textDocument: ITextDocument, + position: Position, + token?: CancellationToken, + options: Exclude<Partial<GetInlineCompletionsOptions>, 'promptOnly'> = {} + ): Promise<CopilotCompletion[] | undefined> { + logCompletionLocation(this.logTargetService, textDocument, position); + + const result = await this.getInlineCompletionsResult(createCompletionState(textDocument, position), token, options); + return this.instantiationService.invokeFunction(handleGhostTextResultTelemetry, result); + } + + private async getInlineCompletionsResult( + completionState: CompletionState, + token?: CancellationToken, + options: GetInlineCompletionsOptions = {} + ): Promise<GhostTextResultWithTelemetry<CopilotCompletion[]>> { + let lineLengthIncrease = 0; + // The golang.go extension (and quite possibly others) uses snippets for function completions, which collapse down + // to look like empty function calls (e.g., `foo()`) in selectedCompletionInfo.text. Injecting that directly into + // the prompt produces low quality completions, so don't. + if (options.selectedCompletionInfo?.text && !options.selectedCompletionInfo.text.includes(')')) { + completionState = completionState.addSelectedCompletionInfo(options.selectedCompletionInfo); + lineLengthIncrease = completionState.position.character - options.selectedCompletionInfo.range.end.character; + } + + const result = await this.instantiationService.invokeFunction(getGhostText, completionState, token, options); + if (result.type !== 'success') { return result; } + const [resultArray, resultType] = result.value; + + if (token?.isCancellationRequested) { + return { + type: 'canceled', + reason: 'after getGhostText', + telemetryData: { telemetryBlob: result.telemetryBlob }, + }; + } + + const index = this.instantiationService.invokeFunction(setLastShown, completionState.textDocument, completionState.position, resultType); + + const completions = completionsFromGhostTextResults( + resultArray, + resultType, + completionState.textDocument, + completionState.position, + options.formattingOptions, + index + ); + if (completions.length === 0) { + // This is a backstop, most/all cases of an empty completions list should be caught earlier + // TODO: figure out how this accounts for 7% of ghostText.empty when it looks unreachable + return { type: 'empty', reason: 'no completions in final result', telemetryData: result.telemetryData }; + } + + // Speculatively request a new completion including the newly returned completion in the document + if (resultType !== ResultType.TypingAsSuggested) { + completionState = completionState.applyEdits([ + { + newText: completions[0].insertText, + range: completions[0].range, + }, + ]); + + // Cache speculative request to be triggered when telemetryShown is called + const specOpts = { isSpeculative: true, opportunityId: options.opportunityId }; + const fn = () => this.instantiationService.invokeFunction(getGhostText, completionState, undefined, specOpts); + this.speculativeRequestCache.set(completions[0].clientCompletionId, fn); + } + + const value = completions.map(completion => { + const { start, end } = completion.range; + const range = Range.create(start, Position.create(end.line, end.character - lineLengthIncrease)); + return { ...completion, range }; + }); + return { ...result, value }; + } +} + +function logCompletionLocation(logTarget: ICompletionsLogTargetService, textDocument: TextDocumentContents, position: Position) { + const prefix = textDocument.getText({ + start: { line: Math.max(position.line - 1, 0), character: 0 }, + end: position, + }); + const suffix = textDocument.getText({ + start: position, + end: { + line: Math.min(position.line + 2, textDocument.lineCount - 1), + character: textDocument.lineCount - 1 > position.line ? 0 : position.character, + }, + }); + + logger.debug( + logTarget, + `Requesting for ${textDocument.uri} at ${position.line}:${position.character}`, + `between ${JSON.stringify(prefix)} and ${JSON.stringify(suffix)}.` + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/language/generatedLanguages.ts b/completions-sample-code/vscode-node/lib/src/language/generatedLanguages.ts new file mode 100644 index 0000000..17d7837 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/language/generatedLanguages.ts @@ -0,0 +1,749 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// This file is generated by running 'npm run generate_languages' +// a map of all known languages (see languageMarkers) with their extensions and filenames as they are defined in linguist +export const knownLanguages: { [language: string]: { extensions: string[]; filenames?: string[] } } = { + abap: { + extensions: ['.abap'], + }, + aspdotnet: { + extensions: ['.asax', '.ascx', '.ashx', '.asmx', '.aspx', '.axd'], + }, + bat: { + extensions: ['.bat', '.cmd'], + }, + bibtex: { + extensions: ['.bib', '.bibtex'], + }, + blade: { + extensions: ['.blade', '.blade.php'], + }, + BluespecSystemVerilog: { + extensions: ['.bsv'], + }, + c: { + extensions: ['.c', '.cats', '.h', '.h.in', '.idc'], + }, + csharp: { + extensions: ['.cake', '.cs', '.cs.pp', '.csx', '.linq'], + }, + cpp: { + extensions: [ + '.c++', + '.cc', + '.cp', + '.cpp', + '.cppm', + '.cxx', + '.h', + '.h++', + '.hh', + '.hpp', + '.hxx', + '.idl', + '.inc', + '.inl', + '.ino', + '.ipp', + '.ixx', + '.rc', + '.re', + '.tcc', + '.tpp', + '.txx', + '.i', + ], + }, + cobol: { + extensions: ['.cbl', '.ccp', '.cob', '.cobol', '.cpy'], + }, + css: { + extensions: ['.css', '.wxss'], + }, + clojure: { + extensions: ['.bb', '.boot', '.cl2', '.clj', '.cljc', '.cljs', '.cljs.hl', '.cljscm', '.cljx', '.edn', '.hic'], + filenames: ['riemann.config'], + }, + ql: { + extensions: ['.ql', '.qll'], + }, + coffeescript: { + extensions: ['._coffee', '.cake', '.cjsx', '.coffee', '.iced'], + filenames: ['Cakefile'], + }, + cuda: { + extensions: ['.cu', '.cuh'], + }, + dart: { + extensions: ['.dart'], + }, + dockerfile: { + extensions: ['.containerfile', '.dockerfile'], + filenames: ['Containerfile', 'Dockerfile'], + }, + dotenv: { + extensions: ['.env'], + filenames: [ + '.env', + '.env.ci', + '.env.dev', + '.env.development', + '.env.development.local', + '.env.example', + '.env.local', + '.env.prod', + '.env.production', + '.env.sample', + '.env.staging', + '.env.test', + '.env.testing', + ], + }, + html: { + extensions: [ + '.ect', + '.ejs', + '.ejs.t', + '.jst', + '.hta', + '.htm', + '.html', + '.html.hl', + '.html5', + '.inc', + '.jsp', + '.njk', + '.tpl', + '.twig', + '.wxml', + '.xht', + '.xhtml', + '.phtml', + '.liquid', + ], + }, + elixir: { + extensions: ['.ex', '.exs'], + filenames: ['mix.lock'], + }, + erlang: { + extensions: ['.app', '.app.src', '.erl', '.es', '.escript', '.hrl', '.xrl', '.yrl'], + filenames: ['Emakefile', 'rebar.config', 'rebar.config.lock', 'rebar.lock'], + }, + fsharp: { + extensions: ['.fs', '.fsi', '.fsx'], + }, + go: { + extensions: ['.go'], + }, + groovy: { + extensions: ['.gradle', '.groovy', '.grt', '.gtpl', '.gvy', '.jenkinsfile'], + filenames: ['Jenkinsfile', 'Jenkinsfile'], + }, + graphql: { + extensions: ['.gql', '.graphql', '.graphqls'], + }, + terraform: { + extensions: ['.hcl', '.nomad', '.tf', '.tfvars', '.workflow'], + }, + hlsl: { + extensions: ['.cginc', '.fx', '.fxh', '.hlsl', '.hlsli'], + }, + erb: { + extensions: ['.erb', '.erb.deface', '.rhtml'], + }, + razor: { + extensions: ['.cshtml', '.razor'], + }, + haml: { + extensions: ['.haml', '.haml.deface'], + }, + handlebars: { + extensions: ['.handlebars', '.hbs'], + }, + haskell: { + extensions: ['.hs', '.hs-boot', '.hsc'], + }, + ini: { + extensions: ['.cfg', '.cnf', '.dof', '.ini', '.lektorproject', '.prefs', '.pro', '.properties', '.url'], + filenames: [ + '.buckconfig', + '.coveragerc', + '.flake8', + '.pylintrc', + 'HOSTS', + 'buildozer.spec', + 'hosts', + 'pylintrc', + 'vlcrc', + ], + }, + json: { + extensions: [ + '.4DForm', + '.4DProject', + '.JSON-tmLanguage', + '.avsc', + '.geojson', + '.gltf', + '.har', + '.ice', + '.json', + '.json.example', + '.jsonl', + '.mcmeta', + '.sarif', + '.tact', + '.tfstate', + '.tfstate.backup', + '.topojson', + '.webapp', + '.webmanifest', + '.yy', + '.yyp', + ], + filenames: [ + '.all-contributorsrc', + '.arcconfig', + '.auto-changelog', + '.c8rc', + '.htmlhintrc', + '.imgbotconfig', + '.nycrc', + '.tern-config', + '.tern-project', + '.watchmanconfig', + 'MODULE.bazel.lock', + 'Package.resolved', + 'Pipfile.lock', + 'bun.lock', + 'composer.lock', + 'deno.lock', + 'flake.lock', + 'mcmod.info', + ], + }, + jsonc: { + extensions: [ + '.code-snippets', + '.code-workspace', + '.jsonc', + '.sublime-build', + '.sublime-color-scheme', + '.sublime-commands', + '.sublime-completions', + '.sublime-keymap', + '.sublime-macro', + '.sublime-menu', + '.sublime-mousemap', + '.sublime-project', + '.sublime-settings', + '.sublime-theme', + '.sublime-workspace', + '.sublime_metrics', + '.sublime_session', + ], + filenames: [ + '.babelrc', + '.devcontainer.json', + '.eslintrc.json', + '.jscsrc', + '.jshintrc', + '.jslintrc', + '.swcrc', + 'api-extractor.json', + 'argv.json', + 'devcontainer.json', + 'extensions.json', + 'jsconfig.json', + 'keybindings.json', + 'language-configuration.json', + 'launch.json', + 'profiles.json', + 'settings.json', + 'tasks.json', + 'tsconfig.json', + 'tslint.json', + ], + }, + java: { + extensions: ['.jav', '.java', '.jsh'], + }, + javascript: { + extensions: [ + '._js', + '.bones', + '.cjs', + '.es', + '.es6', + '.frag', + '.gs', + '.jake', + '.javascript', + '.js', + '.jsb', + '.jscad', + '.jsfl', + '.jslib', + '.jsm', + '.jspre', + '.jss', + '.mjs', + '.njs', + '.pac', + '.sjs', + '.ssjs', + '.xsjs', + '.xsjslib', + ], + filenames: ['Jakefile'], + }, + julia: { + extensions: ['.jl'], + }, + kotlin: { + extensions: ['.kt', '.ktm', '.kts'], + }, + less: { + extensions: ['.less'], + }, + lua: { + extensions: ['.fcgi', '.lua', '.luau', '.nse', '.p8', '.pd_lua', '.rbxs', '.rockspec', '.wlua'], + filenames: ['.luacheckrc'], + }, + makefile: { + extensions: ['.d', '.mak', '.make', '.makefile', '.mk', '.mkfile'], + filenames: [ + 'BSDmakefile', + 'GNUmakefile', + 'Kbuild', + 'Makefile', + 'Makefile.am', + 'Makefile.boot', + 'Makefile.frag', + 'Makefile.in', + 'Makefile.inc', + 'Makefile.wat', + 'makefile', + 'makefile.sco', + 'mkfile', + ], + }, + markdown: { + extensions: [ + '.livemd', + '.markdown', + '.md', + '.mdown', + '.mdwn', + '.mdx', + '.mkd', + '.mkdn', + '.mkdown', + '.ronn', + '.scd', + '.workbook', + ], + filenames: ['contents.lr'], + }, + 'objective-c': { + extensions: ['.h', '.m'], + }, + 'objective-cpp': { + extensions: ['.mm'], + }, + php: { + extensions: [ + '.aw', + '.ctp', + '.fcgi', + '.inc', + '.install', + '.module', + '.php', + '.php3', + '.php4', + '.php5', + '.phps', + '.phpt', + '.theme', + ], + filenames: ['.php', '.php_cs', '.php_cs.dist', 'Phakefile'], + }, + perl: { + extensions: ['.al', '.cgi', '.fcgi', '.perl', '.ph', '.pl', '.plx', '.pm', '.psgi', '.t'], + filenames: ['.latexmkrc', 'Makefile.PL', 'Rexfile', 'ack', 'cpanfile', 'latexmkrc'], + }, + powershell: { + extensions: ['.ps1', '.psd1', '.psm1'], + }, + pug: { + extensions: ['.jade', '.pug'], + }, + python: { + extensions: [ + '.cgi', + '.codon', + '.fcgi', + '.gyp', + '.gypi', + '.lmi', + '.py', + '.py3', + '.pyde', + '.pyi', + '.pyp', + '.pyt', + '.pyw', + '.rpy', + '.sage', + '.spec', + '.tac', + '.wsgi', + '.xpy', + ], + filenames: ['.gclient', 'DEPS', 'SConscript', 'SConstruct', 'wscript'], + }, + r: { + extensions: ['.r', '.rd', '.rsx'], + filenames: ['.Rprofile', 'expr-dist'], + }, + ruby: { + extensions: [ + '.builder', + '.eye', + '.fcgi', + '.gemspec', + '.god', + '.jbuilder', + '.mspec', + '.pluginspec', + '.podspec', + '.prawn', + '.rabl', + '.rake', + '.rb', + '.rbi', + '.rbuild', + '.rbw', + '.rbx', + '.ru', + '.ruby', + '.spec', + '.thor', + '.watchr', + ], + filenames: [ + '.irbrc', + '.pryrc', + '.simplecov', + 'Appraisals', + 'Berksfile', + 'Brewfile', + 'Buildfile', + 'Capfile', + 'Dangerfile', + 'Deliverfile', + 'Fastfile', + 'Gemfile', + 'Guardfile', + 'Jarfile', + 'Mavenfile', + 'Podfile', + 'Puppetfile', + 'Rakefile', + 'Snapfile', + 'Steepfile', + 'Thorfile', + 'Vagrantfile', + 'buildfile', + ], + }, + rust: { + extensions: ['.rs', '.rs.in'], + }, + scss: { + extensions: ['.scss'], + }, + sql: { + extensions: ['.cql', '.ddl', '.inc', '.mysql', '.prc', '.sql', '.tab', '.udf', '.viw'], + }, + sass: { + extensions: ['.sass'], + }, + scala: { + extensions: ['.kojo', '.sbt', '.sc', '.scala'], + }, + shellscript: { + extensions: [ + '.bash', + '.bats', + '.cgi', + '.command', + '.fcgi', + '.fish', + '.ksh', + '.sh', + '.sh.in', + '.tmux', + '.tool', + '.trigger', + '.zsh', + '.zsh-theme', + ], + filenames: [ + '.bash_aliases', + '.bash_functions', + '.bash_history', + '.bash_logout', + '.bash_profile', + '.bashrc', + '.cshrc', + '.envrc', + '.flaskenv', + '.kshrc', + '.login', + '.profile', + '.tmux.conf', + '.zlogin', + '.zlogout', + '.zprofile', + '.zshenv', + '.zshrc', + '9fs', + 'PKGBUILD', + 'bash_aliases', + 'bash_logout', + 'bash_profile', + 'bashrc', + 'cshrc', + 'gradlew', + 'kshrc', + 'login', + 'man', + 'profile', + 'tmux.conf', + 'zlogin', + 'zlogout', + 'zprofile', + 'zshenv', + 'zshrc', + ], + }, + slang: { + extensions: ['.fxc', '.hlsl', '.s', '.slang', '.slangh', '.usf', '.ush', '.vfx'], + }, + slim: { + extensions: ['.slim'], + }, + solidity: { + extensions: ['.sol'], + }, + stylus: { + extensions: ['.styl'], + }, + svelte: { + extensions: ['.svelte'], + }, + swift: { + extensions: ['.swift'], + }, + systemverilog: { + extensions: ['.sv', '.svh', '.vh'], + }, + typescriptreact: { + extensions: ['.tsx'], + }, + latex: { + extensions: [ + '.aux', + '.bbx', + '.cbx', + '.cls', + '.dtx', + '.ins', + '.lbx', + '.ltx', + '.mkii', + '.mkiv', + '.mkvi', + '.sty', + '.tex', + '.toc', + ], + }, + typescript: { + extensions: ['.cts', '.mts', '.ts'], + }, + verilog: { + extensions: ['.v', '.veo'], + }, + vim: { + extensions: ['.vba', '.vim', '.vimrc', '.vmb'], + filenames: ['.exrc', '.gvimrc', '.nvimrc', '.vimrc', '_vimrc', 'gvimrc', 'nvimrc', 'vimrc'], + }, + vb: { + extensions: ['.vb', '.vbhtml', '.Dsr', '.bas', '.cls', '.ctl', '.frm', '.vbs'], + }, + vue: { + extensions: ['.nvue', '.vue'], + }, + xml: { + extensions: [ + '.adml', + '.admx', + '.ant', + '.axaml', + '.axml', + '.builds', + '.ccproj', + '.ccxml', + '.clixml', + '.cproject', + '.cscfg', + '.csdef', + '.csl', + '.csproj', + '.ct', + '.depproj', + '.dita', + '.ditamap', + '.ditaval', + '.dll.config', + '.dotsettings', + '.filters', + '.fsproj', + '.fxml', + '.glade', + '.gml', + '.gmx', + '.gpx', + '.grxml', + '.gst', + '.hzp', + '.iml', + '.ivy', + '.jelly', + '.jsproj', + '.kml', + '.launch', + '.mdpolicy', + '.mjml', + '.mod', + '.mojo', + '.mxml', + '.natvis', + '.ncl', + '.ndproj', + '.nproj', + '.nuspec', + '.odd', + '.osm', + '.pkgproj', + '.plist', + '.pluginspec', + '.proj', + '.props', + '.ps1xml', + '.psc1', + '.pt', + '.pubxml', + '.qhelp', + '.rdf', + '.res', + '.resx', + '.rss', + '.sch', + '.scxml', + '.sfproj', + '.shproj', + '.srdf', + '.storyboard', + '.sublime-snippet', + '.svg', + '.sw', + '.targets', + '.tml', + '.typ', + '.ui', + '.urdf', + '.ux', + '.vbproj', + '.vcxproj', + '.vsixmanifest', + '.vssettings', + '.vstemplate', + '.vxml', + '.wixproj', + '.workflow', + '.wsdl', + '.wsf', + '.wxi', + '.wxl', + '.wxs', + '.x3d', + '.xacro', + '.xaml', + '.xib', + '.xlf', + '.xliff', + '.xmi', + '.xml', + '.xml.dist', + '.xmp', + '.xproj', + '.xsd', + '.xspec', + '.xul', + '.zcml', + ], + filenames: [ + '.classpath', + '.cproject', + '.project', + 'App.config', + 'NuGet.config', + 'Settings.StyleCop', + 'Web.Debug.config', + 'Web.Release.config', + 'Web.config', + 'packages.config', + ], + }, + xsl: { + extensions: ['.xsl', '.xslt'], + }, + yaml: { + extensions: [ + '.mir', + '.reek', + '.rviz', + '.sublime-syntax', + '.syntax', + '.yaml', + '.yaml-tmlanguage', + '.yaml.sed', + '.yml', + '.yml.mysql', + ], + filenames: [ + '.clang-format', + '.clang-tidy', + '.clangd', + '.gemrc', + 'CITATION.cff', + 'glide.lock', + 'pixi.lock', + 'yarn.lock', + ], + }, + javascriptreact: { + extensions: ['.jsx'], + }, + legend: { + extensions: ['.pure'], + }, +}; diff --git a/completions-sample-code/vscode-node/lib/src/language/languageDetection.ts b/completions-sample-code/vscode-node/lib/src/language/languageDetection.ts new file mode 100644 index 0000000..6ecd89b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/language/languageDetection.ts @@ -0,0 +1,147 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { knownLanguages } from './generatedLanguages'; +import { + knownFileExtensions, + knownTemplateLanguageExtensions, + templateLanguageLimitations, +} from './languages'; +import { basename } from '../util/uri'; +import * as path from 'node:path'; + +export class Language { + constructor( + readonly languageId: string, + readonly isGuess: boolean, + readonly fileExtension: string + ) { } +} + +interface LanguageDetectionInput { + languageId: string; + uri: string; +} + +export abstract class LanguageDetection { + abstract detectLanguage(doc: LanguageDetectionInput): Language; +} + +type LanguageIdWithGuessing = { languageId: string; isGuess: boolean }; + +const knownExtensions = new Map<string, string[]>(); +const knownFilenames = new Map<string, string[]>(); + +for (const [languageId, { extensions, filenames }] of Object.entries(knownLanguages)) { + for (const extension of extensions) { + knownExtensions.set(extension, [...(knownExtensions.get(extension) ?? []), languageId]); + } + for (const filename of filenames ?? []) { + knownFilenames.set(filename, [...(knownFilenames.get(filename) ?? []), languageId]); + } +} + +class FilenameAndExensionLanguageDetection extends LanguageDetection { + detectLanguage(doc: LanguageDetectionInput): Language { + const filename = basename(doc.uri); + const extension = path.extname(filename).toLowerCase(); + const extensionWithoutTemplate = this.extensionWithoutTemplateLanguage(filename, extension); + const languageIdWithGuessing = this.detectLanguageId(filename, extensionWithoutTemplate); + const ext = this.computeFullyQualifiedExtension(extension, extensionWithoutTemplate); + if (!languageIdWithGuessing) { + return new Language(doc.languageId, true, ext); + } + return new Language(languageIdWithGuessing.languageId, languageIdWithGuessing.isGuess, ext); + } + + private extensionWithoutTemplateLanguage(filename: string, extension: string): string { + if (knownTemplateLanguageExtensions.includes(extension)) { + const filenameWithoutExtension = filename.substring(0, filename.lastIndexOf('.')); + const extensionWithoutTemplate = path.extname(filenameWithoutExtension).toLowerCase(); + const isTemplateLanguage = + extensionWithoutTemplate.length > 0 && + knownFileExtensions.includes(extensionWithoutTemplate) && + this.isExtensionValidForTemplateLanguage(extension, extensionWithoutTemplate); + if (isTemplateLanguage) { + return extensionWithoutTemplate; + } + } + return extension; + } + + private isExtensionValidForTemplateLanguage(extension: string, extensionWithoutTemplate: string): boolean { + const limitations = templateLanguageLimitations[extension]; + return !limitations || limitations.includes(extensionWithoutTemplate); + } + + private detectLanguageId(filename: string, extension: string): LanguageIdWithGuessing | undefined { + if (knownFilenames.has(filename)) { + return { languageId: knownFilenames.get(filename)![0], isGuess: false }; + } + const extensionCandidates = knownExtensions.get(extension) ?? []; + if (extensionCandidates.length > 0) { + return { languageId: extensionCandidates[0], isGuess: extensionCandidates.length > 1 }; + } + while (filename.includes('.')) { + filename = filename.replace(/\.[^.]*$/, ''); + if (knownFilenames.has(filename)) { + return { languageId: knownFilenames.get(filename)![0], isGuess: false }; + } + } + } + + private computeFullyQualifiedExtension(extension: string, extensionWithoutTemplate: string): string { + if (extension !== extensionWithoutTemplate) { + return extensionWithoutTemplate + extension; + } + return extension; + } +} + +// This class is used to group similar languages together. +// The main drawback of trying to keep them apart is that for related files (e.g. header files), +// the language detection might be wrong and thus features like neighbor tabs might not work as expected. +// In the end, this feature should be moved to neighborTabs.ts (but that's hard to do behind a feature flag) +class GroupingLanguageDetection extends LanguageDetection { + constructor(private readonly delegate: LanguageDetection) { + super(); + } + + detectLanguage(doc: LanguageDetectionInput): Language { + const language = this.delegate.detectLanguage(doc); + const languageId = language.languageId; + if (languageId === 'c' || languageId === 'cpp') { + return new Language('cpp', language.isGuess, language.fileExtension); + } + return language; + } +} + +class ClientProvidedLanguageDetection extends LanguageDetection { + constructor(private readonly delegate: LanguageDetection) { + super(); + } + + detectLanguage(doc: LanguageDetectionInput): Language { + if (doc.uri.startsWith('untitled:') || doc.uri.startsWith('vscode-notebook-cell:')) { + return new Language(doc.languageId, true, ''); + } + return this.delegate.detectLanguage(doc); + } +} + +export const languageDetection = new GroupingLanguageDetection( + new ClientProvidedLanguageDetection(new FilenameAndExensionLanguageDetection()) +); + +export function detectLanguage({ uri, languageId }: { uri: string; languageId: string }): string; +export function detectLanguage({ uri }: { uri: string }): string | undefined; +export function detectLanguage({ uri, languageId }: { uri: string; languageId?: string }) { + const language = languageDetection.detectLanguage({ uri, languageId: 'UNKNOWN' }); + if (language.languageId === 'UNKNOWN') { + return languageId; + } + return language.languageId; +} diff --git a/completions-sample-code/vscode-node/lib/src/language/languages.ts b/completions-sample-code/vscode-node/lib/src/language/languages.ts new file mode 100644 index 0000000..d0b65bd --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/language/languages.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { knownLanguages } from './generatedLanguages'; + +export const knownTemplateLanguageExtensions = [ + '.ejs', + '.erb', + '.haml', + '.hbs', + '.j2', + '.jinja', + '.jinja2', + '.liquid', + '.mustache', + '.njk', + '.php', + '.pug', + '.slim', + '.webc', +]; + +export const templateLanguageLimitations: { [extension: string]: string[] } = { + '.php': ['.blade'], +}; + +export type LanguageInfo = { + extensions: string[]; + filenames?: string[]; +}; + +export const knownFileExtensions = Object.keys(knownLanguages).flatMap(language => knownLanguages[language].extensions); \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/language/test/generatedLanguages.test.ts b/completions-sample-code/vscode-node/lib/src/language/test/generatedLanguages.test.ts new file mode 100644 index 0000000..acfdf1a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/language/test/generatedLanguages.test.ts @@ -0,0 +1,26 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { knownLanguages } from '../generatedLanguages'; +import { languageMarkers } from '../../../../prompt/src/languageMarker'; +import * as assert from 'assert'; + +suite('generated languages', function () { + // tex exists as latex and tex in language markers + // jsx exists as jsx and javascriptreact in language markers. However jsx is never detected according to telemetry data + // vue-html will be detected as html + const ignoredMappings = ['jsx', 'tex', 'vue-html']; + + for (const marker in languageMarkers) { + if (!ignoredMappings.includes(marker)) { + test(`'${marker}' is generated`, function () { + assert.ok( + marker in knownLanguages, + 'language for comment marker ' + marker + ' has not been generated' + ); + }); + } + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/language/test/languageDetection.test.ts b/completions-sample-code/vscode-node/lib/src/language/test/languageDetection.test.ts new file mode 100644 index 0000000..fdce676 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/language/test/languageDetection.test.ts @@ -0,0 +1,212 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { createTextDocument } from '../../test/textDocument'; +import { Language, LanguageDetection, languageDetection } from '../languageDetection'; + +suite('language detection', function () { + test('reuse languages for untitled documents', function () { + assert.deepStrictEqual( + languageDetection.detectLanguage({ uri: 'untitled:///abc', languageId: 'typescript' }), + new Language('typescript', true, '') + ); + }); + + test('normalizes "c" to "cpp" for untitled documents', function () { + assert.deepStrictEqual( + languageDetection.detectLanguage({ uri: 'untitled:///abc', languageId: 'c' }), + new Language('cpp', true, '') + ); + }); + + test('reuse languages for notebook documents', function () { + assert.deepStrictEqual( + languageDetection.detectLanguage({ uri: 'vscode-notebook-cell:/abc', languageId: 'typescript' }).languageId, + 'typescript' + ); + }); + + const toDetectByExtension: [string, string][] = [ + ['.ts', 'typescript'], + ['.js', 'javascript'], + ['.jsx', 'javascriptreact'], + ['.tsx', 'typescriptreact'], + ['.html', 'html'], + ['.html5', 'html'], + ['.css', 'css'], + ['.scss', 'scss'], + ['.less', 'less'], + ['.jsonc', 'jsonc'], + ['.json', 'json'], + ['.xml', 'xml'], + ['.yml', 'yaml'], + ['.yaml', 'yaml'], + ['.php', 'php'], + ['.py', 'python'], + ['.rb', 'ruby'], + ['.go', 'go'], + ['.java', 'java'], + ['.cs', 'csharp'], + ['.cpp', 'cpp'], + ['.c', 'cpp'], + ['.C', 'cpp'], + ['.h', 'cpp'], + ['.sh', 'shellscript'], + ['.bash', 'shellscript'], + ['.sql', 'sql'], + ['.swift', 'swift'], + ['.vb', 'vb'], + ['.frm', 'vb'], + ['.lua', 'lua'], + ['.tex', 'latex'], + ['.md', 'markdown'], + ['.markdown', 'markdown'], + ['.r', 'r'], + ['.R', 'r'], + ['.blade.php', 'blade'], + ['.BLADE.php', 'blade'], + ['.gradle', 'groovy'], + ['.gradle.kts', 'kotlin'], + ['.ejs', 'html'], + ['.liquid', 'html'], + ['.yml.erb', 'yaml'], + ['.yml.njk', 'yaml'], + ['.some.file.yml.njk', 'yaml'], + ['.phtml', 'html'], + ['f.sourcecode.php', 'php'], + ['.plist', 'xml'], + ['.svg', 'xml'], + ['.jsp', 'html'], + ['.code-workspace', 'jsonc'], + ['.wxss', 'css'], + ['.luau', 'lua'], + ['.codon', 'python'], + ['.edn', 'clojure'], + ['.tpl', 'html'], + ['.rs', 'rust'], + ['.bas', 'vb'], + ['.wxml', 'html'], + ['.nvue', 'vue'], + ['.jenkinsfile', 'groovy'], + ['.twig', 'html'], + ['.inc.php', 'php'], + ['.mm', 'objective-cpp'], + ['.module', 'php'], + ['.install', 'php'], + ['.theme', 'php'], + ['.rc', 'cpp'], + ['.idl', 'cpp'], + ['.pubxml', 'xml'], + ['.njk', 'html'], + ['.fish', 'shellscript'], + ['.vbs', 'vb'], + ['.sage', 'python'], + ['.mdx', 'markdown'], + ['.somethingelse', 'clientProvidedLanguageId'], + ]; + + toDetectByExtension.forEach(([extension, languageId]) => { + test(`detect ${languageId} by file extension ${extension}`, function () { + assertLanguageId(`file:///test${extension}`, languageId); + }); + }); + + const toDetectByFilename: [string, string][] = [ + ['.bash_history', 'shellscript'], + ['.bashrc', 'shellscript'], + ['.zshrc', 'shellscript'], + ['.irbrc', 'ruby'], + ['Gemfile', 'ruby'], + ['riemann.config', 'clojure'], + ['Dockerfile', 'dockerfile'], + ['Dockerfile.local', 'dockerfile'], + ['.env.production', 'dotenv'], + ['.env.development.local', 'dotenv'], + ['Jenkinsfile', 'groovy'], + ['Makefile', 'makefile'], + ['.classpath', 'xml'], + ['.gemrc', 'yaml'], + ['tsconfig.json', 'jsonc'], + ['.eslintrc.json', 'jsonc'], + ['settings.json', 'jsonc'], + ['tasks.json', 'jsonc'], + ['keybindings.json', 'jsonc'], + ['extensions.json', 'jsonc'], + ['argv.json', 'jsonc'], + ['profiles.json', 'jsonc'], + ['devcontainer.json', 'jsonc'], + ['.devcontainer.json', 'jsonc'], + ]; + + toDetectByFilename.forEach(([filename, languageId]) => { + test(`detect ${languageId} by filename ${filename}`, function () { + assertLanguageId(`file:///${filename}`, languageId); + }); + }); + + const urls: [string, string][] = [ + ['file:///some/path/test.ts', 'typescript'], + ['untitled:///some/path/test', 'clientProvidedLanguageId'], + ['file:////server-name/shared-resource-pathname/test.sh', 'shellscript'], + ]; + + urls.forEach(([url, languageId]) => { + test(`detect ${languageId} by url ${url}`, function () { + assertLanguageId(url, languageId); + }); + }); + + const extensionsToDetect: [string, string][] = [ + ['', ''], + ['.ts', '.ts'], + ['a.longer.path.ts', '.ts'], + ['.sh', '.sh'], + ['.html.erb', '.html.erb'], + ['.html.slim', '.html.slim'], + ['.unknown.erb', '.erb'], + ['.yaml.njk', '.yaml.njk'], + ['.unknown', '.unknown'], + ]; + + extensionsToDetect.forEach(([filename, extension]) => { + test(`detect extension ${extension} by filename test${filename}`, function () { + assertExtension(`file:///test${filename}`, extension); + }); + }); + + test(`has no extension for filename without extension`, function () { + assertExtension(`file:///.secretproduct`, ''); + }); + + function assertExtension(uri: string, expectedExtension: string) { + const doc = createTextDocument(uri, 'clientProvidedLanguageId', 1, 'test content'); + + const language = languageDetection.detectLanguage(doc); + + assert.deepStrictEqual(language.fileExtension, expectedExtension); + } + + function assertLanguageId(uri: string, expectedLanguageId: string) { + const doc = createTextDocument(uri, 'clientProvidedLanguageId', 1, 'test content'); + + const language = languageDetection.detectLanguage(doc); + + assert.deepStrictEqual(language.languageId, expectedLanguageId); + } + + test('detected languages for ambiguous options will be re-detected', function () { + assert.deepStrictEqual(detect('testfile.c', languageDetection).languageId, 'cpp'); + assert.deepStrictEqual(detect('testfile.h', languageDetection).languageId, 'cpp'); + assert.deepStrictEqual(detect('testfile.cpp', languageDetection).languageId, 'cpp'); + assert.deepStrictEqual(detect('testfile.h', languageDetection).languageId, 'cpp'); + }); + + function detect(filename: string, languageDetection: LanguageDetection): Language { + return languageDetection.detectLanguage( + createTextDocument(`file:///${filename}`, 'clientProvidedLanguageId', 1, 'test content') + ); + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/localFileSystem.ts b/completions-sample-code/vscode-node/lib/src/localFileSystem.ts new file mode 100644 index 0000000..5b82f39 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/localFileSystem.ts @@ -0,0 +1,67 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { Stats, promises as fsp } from 'fs'; +import { join } from 'path'; +import { FileIdentifier, FileStat, FileType, ICompletionsFileSystemService } from './fileSystem'; +import { fsPath } from './util/uri'; + +export class LocalFileSystem implements ICompletionsFileSystemService { + declare _serviceBrand: undefined; + + async readFileString(uri: FileIdentifier): Promise<string> { + return (await fsp.readFile(fsPath(uri))).toString(); + } + + async stat(uri: FileIdentifier): Promise<FileStat> { + const { targetStat, lstat, stat } = await this.statWithLink(fsPath(uri)); + return { + ctime: targetStat.ctimeMs, + mtime: targetStat.mtimeMs, + size: targetStat.size, + type: this.getFileType(targetStat, lstat, stat), + }; + } + + async readDirectory(uri: FileIdentifier): Promise<[string, FileType][]> { + const filePath = fsPath(uri); + const readDir = await fsp.readdir(filePath, { withFileTypes: true }); + const result: [string, FileType][] = []; + for (const file of readDir) { + const { targetStat, lstat, stat } = await this.statWithLink(join(filePath, file.name)); + result.push([file.name, this.getFileType(targetStat, lstat, stat)]); + } + return result; + } + + private async statWithLink(fsPath: string): Promise<{ lstat: Stats; stat?: Stats; targetStat: Stats }> { + const lstat = await fsp.lstat(fsPath); + + if (lstat.isSymbolicLink()) { + try { + const stat = await fsp.stat(fsPath); + return { lstat, stat, targetStat: stat }; + } catch { + // likely a dangling link or access error + } + } + + return { lstat, targetStat: lstat }; + } + + private getFileType(targetStat: Stats, lstat: Stats, stat?: Stats): FileType { + let type = FileType.Unknown; + if (targetStat.isFile()) { + type = FileType.File; + } + if (targetStat.isDirectory()) { + type = FileType.Directory; + } + // dangling links have FileType.Unknown + if (lstat.isSymbolicLink() && stat) { + type |= FileType.SymbolicLink; + } + return type; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/logger.ts b/completions-sample-code/vscode-node/lib/src/logger.ts new file mode 100644 index 0000000..f6da30d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/logger.ts @@ -0,0 +1,83 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** + * This file is kept with minimal dependencies to avoid circular dependencies + * breaking module resolution since the Logger class is instantiated at the + * module level in many places. + * + * Do not add any concrete dependencies here. + */ +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTelemetryService } from '../../bridge/src/completionsTelemetryServiceBridge'; +import { telemetryException } from './telemetry'; + +export enum LogLevel { + DEBUG = 4, + INFO = 3, + WARN = 2, + ERROR = 1, +} + +export const ICompletionsLogTargetService = createServiceIdentifier<ICompletionsLogTargetService>('ICompletionsLogTargetService'); +export interface ICompletionsLogTargetService { + readonly _serviceBrand: undefined; + logIt(level: LogLevel, category: string, ...extra: unknown[]): void; +} + +export class Logger { + constructor(private readonly category: string) { } + + private log(logTarget: ICompletionsLogTargetService, level: LogLevel, ...extra: unknown[]) { + logTarget.logIt(level, this.category, ...extra); + } + + debug(logTarget: ICompletionsLogTargetService, ...extra: unknown[]) { + this.log(logTarget, LogLevel.DEBUG, ...extra); + } + + info(logTarget: ICompletionsLogTargetService, ...extra: unknown[]) { + this.log(logTarget, LogLevel.INFO, ...extra); + } + + warn(logTarget: ICompletionsLogTargetService, ...extra: unknown[]) { + this.log(logTarget, LogLevel.WARN, ...extra); + } + + /** + * Logs an error message and reports an error to telemetry. This is appropriate for generic + * error logging, which might not be associated with an exception. Prefer `exception()` when + * logging exception details. + */ + error(logTarget: ICompletionsLogTargetService, ...extra: unknown[]) { + this.log(logTarget, LogLevel.ERROR, ...extra); + } + + /** + * Logs an error message and reports the exception to telemetry. Prefer this method over + * `error()` when logging exception details. + * + * @param accessor The accessor + * @param error The Error object that was thrown + * @param message An optional message for context (e.g. "Request error"). Must not contain customer data. **Do not include stack trace or messages from the error object.** + */ + exception(accessor: ServicesAccessor, error: unknown, origin: string) { + // ignore VS Code cancellations + if (error instanceof Error && error.name === 'Canceled' && error.message === 'Canceled') { return; } + + let message = origin; + if (origin.startsWith('.')) { + message = origin.substring(1); + origin = `${this.category}${origin}`; + } + + telemetryException(accessor.get(ICompletionsTelemetryService), error, origin); + + const safeError: Error = error instanceof Error ? error : new Error(`Non-error thrown: ${String(error)}`); + this.log(accessor.get(ICompletionsLogTargetService), LogLevel.ERROR, `${message}:`, safeError); + } +} + +export const logger = new Logger('default'); diff --git a/completions-sample-code/vscode-node/lib/src/logging/util.ts b/completions-sample-code/vscode-node/lib/src/logging/util.ts new file mode 100644 index 0000000..259a71e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/logging/util.ts @@ -0,0 +1,14 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import util from 'node:util'; + +export function formatLogMessage(category: string, ...extra: unknown[]): string { + return `[${category}] ${format(extra)}`; +} + +function format(args: unknown[]): string { + return util.formatWithOptions({ maxStringLength: Infinity }, ...args); +} diff --git a/completions-sample-code/vscode-node/lib/src/networkConfiguration.ts b/completions-sample-code/vscode-node/lib/src/networkConfiguration.ts new file mode 100644 index 0000000..bbafe29 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/networkConfiguration.ts @@ -0,0 +1,83 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { IAuthenticationService } from '../../../../../platform/authentication/common/authentication'; +import { ICAPIClientService } from '../../../../../platform/endpoint/common/capiClient'; +import { ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotToken } from './auth/copilotTokenManager'; +import { BuildInfo, ConfigKey, ConfigKeyType, getConfig } from './config'; +import { ICompletionsRuntimeModeService } from './util/runtimeMode'; +import { joinPath } from './util/uri'; + +type ServiceEndpoints = { + proxy: string; + 'origin-tracker': string; +}; + +function getDefaultEndpoints(accessor: ServicesAccessor): ServiceEndpoints { + const capi = accessor.get(ICAPIClientService); + return { + proxy: capi.proxyBaseURL, + 'origin-tracker': capi.originTrackerURL, + }; +} + +/** + * If a configuration value has been configured for any of `overrideKeys`, returns + * that value. If `testOverrideKeys` is supplied and the run mode is test, + * `testOverrideKeys` is used instead of `overrideKeys`. + */ +function urlConfigOverride( + accessor: ServicesAccessor, + overrideKeys: ConfigKeyType[], + testOverrideKeys?: ConfigKeyType[] +): string | undefined { + if (testOverrideKeys !== undefined && accessor.get(ICompletionsRuntimeModeService).isRunningInTest()) { + for (const overrideKey of testOverrideKeys) { + const override = getConfig<string>(accessor, overrideKey); + if (override) { return override; } + } + return undefined; + } + + for (const overrideKey of overrideKeys) { + const override = getConfig<string>(accessor, overrideKey); + if (override) { return override; } + } + return undefined; +} + +function getEndpointOverrideUrl(accessor: ServicesAccessor, endpoint: keyof ServiceEndpoints): string | undefined { + switch (endpoint) { + case 'proxy': + return urlConfigOverride( + accessor, + [ConfigKey.DebugOverrideProxyUrl, ConfigKey.DebugOverrideProxyUrlLegacy], + [ConfigKey.DebugTestOverrideProxyUrl, ConfigKey.DebugTestOverrideProxyUrlLegacy] + ); + case 'origin-tracker': + if (!BuildInfo.isProduction()) { + return urlConfigOverride(accessor, [ConfigKey.DebugSnippyOverrideUrl]); + } + } +} + +export function getEndpointUrl( + accessor: ServicesAccessor, + token: CopilotToken, + endpoint: keyof ServiceEndpoints, + ...paths: string[] +): string { + const root = getEndpointOverrideUrl(accessor, endpoint) ?? (token.endpoints ? token.endpoints[endpoint] : undefined) ?? getDefaultEndpoints(accessor)[endpoint]; + return joinPath(root, ...paths); +} + +/** + * Return the endpoints from the most recent token, or fall back to the defaults if we don't have one. + * Generally you should be using token.endpoints or getEndpointUrl() instead. + */ +export function getLastKnownEndpoints(accessor: ServicesAccessor) { + return accessor.get(IAuthenticationService).copilotToken?.endpoints ?? getDefaultEndpoints(accessor); +} + diff --git a/completions-sample-code/vscode-node/lib/src/networking.ts b/completions-sample-code/vscode-node/lib/src/networking.ts new file mode 100644 index 0000000..b466949 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/networking.ts @@ -0,0 +1,170 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CancellationToken } from '../../types/src'; +import { apiVersion, editorVersionHeaders } from './config'; +import { telemetry, TelemetryData } from './telemetry'; + +/** + * CIRCULAR DEPENDENCY FIX - PROGRESSIVE REFACTORING + * + * This module was refactored to resolve a circular dependency that caused runtime errors: + * + * Previous circular dependency chain: + * networking.ts รขโ€ โ€™ config.ts รขโ€ โ€™ features.ts รขโ€ โ€™ copilotTokenManager.ts รขโ€ โ€™ copilotToken.ts รขโ€ โ€™ github.ts รขโ€ โ€™ networking.ts + * + * The issue: + * - networking.ts defined FetchResponseError and other error classes + * - network/github.ts needed FetchResponseError, so imported from networking.ts + * - But networking.ts indirectly depended on github.ts through the config chain + * - This caused "Cannot access 'FetchResponseError' before initialization" runtime error + * + * Solution - Module Separation: + * 1. Extracted all error classes and types to '#lib/networking/networkingTypes' + * 2. github.ts now imports FetchResponseError directly from the types module + * 3. This breaks the circular dependency while preserving functionality + * 4. No more dynamic imports needed since errors and types are in the same module + * + * Progressive Refactoring Strategy: + * - Re-export everything from the new module to maintain API compatibility + * - 22+ files across the codebase import from './networking' and expect these exports + * - This approach allows internal restructuring without breaking existing imports + * - Future: Could gradually migrate files to import directly from networkingTypes module + */ + +// Re-export everything from networking types module for backward compatibility +export * from './networkingTypes'; + +// Import what we need locally for this module's implementation +import { ConfigKey, IConfigurationService } from '../../../../../platform/configuration/common/configurationService'; +import { IEnvService } from '../../../../../platform/env/common/envService'; +import { IFetcherService } from '../../../../../platform/networking/common/fetcherService'; +import { IExperimentationService } from '../../../../../platform/telemetry/common/nullExperimentationService'; +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { FetchOptions, ReqHeaders, Response } from './networkingTypes'; + +export const ICompletionsFetcherService = createServiceIdentifier<ICompletionsFetcherService>('ICompletionsFetcherService'); +export interface ICompletionsFetcherService { + readonly _serviceBrand: undefined; + getImplementation(): ICompletionsFetcherService | Promise<ICompletionsFetcherService>; + fetch(url: string, options: FetchOptions): Promise<Response>; + disconnectAll(): Promise<unknown>; +} + +export class CompletionsFetcher implements ICompletionsFetcherService { + declare _serviceBrand: undefined; + + constructor( + @IConfigurationService private readonly configurationService: IConfigurationService, + @IFetcherService private readonly fetcherService: IFetcherService, + @IExperimentationService private readonly experimentationService: IExperimentationService + ) { } + + getImplementation(): ICompletionsFetcherService | Promise<ICompletionsFetcherService> { + return this; + } + + fetch(url: string, options: FetchOptions): Promise<Response> { + const useFetcher = this.configurationService.getExperimentBasedConfig(ConfigKey.CompletionsFetcher, this.experimentationService) || undefined; + return this.fetcherService.fetch(url, useFetcher ? { ...options, useFetcher } : options); + } + disconnectAll(): Promise<unknown> { + return this.fetcherService.disconnectAll(); + } +} + +/** + * Encapsulates all the functionality related to making GET/POST/DELETE requests using + * different libraries (and in the future, different environments like web vs + * node). + */ +export abstract class Fetcher { + abstract readonly name: string; + /** + * Returns the real implementation, not a delegator. Used by diagnostics to ensure the fetcher name and all + * reachability checks are aligned. + */ + getImplementation(): Fetcher | Promise<Fetcher> { + return this; + } + abstract fetch(url: string, options: FetchOptions): Promise<Response>; + abstract disconnectAll(): Promise<unknown>; +} + +export function postRequest( + accessor: ServicesAccessor, + url: string, + secretKey: string, + intent: string | undefined, // Must be passed in, even if explicitly `undefined` + requestId: string, + body?: Record<string, unknown>, + cancelToken?: CancellationToken, + extraHeaders?: Record<string, string>, + timeout?: number, + modelProviderName?: string +): Promise<Response> { + const fetcher = accessor.get(ICompletionsFetcherService); + const instantiationService = accessor.get(IInstantiationService); + + const headers: ReqHeaders = { + ...extraHeaders, + Authorization: `Bearer ${secretKey}`, + ...instantiationService.invokeFunction(editorVersionHeaders), + }; + + // If we call byok endpoint, no need to add these headers + if (modelProviderName === undefined) { + headers['Openai-Organization'] = 'github-copilot'; + headers['X-Request-Id'] = requestId; + headers['VScode-SessionId'] = accessor.get(IEnvService).sessionId; + headers['VScode-MachineId'] = accessor.get(IEnvService).machineId; + headers['X-GitHub-Api-Version'] = apiVersion; + } + + if (intent) { + headers['OpenAI-Intent'] = intent; + } + + const request: FetchOptions = { + method: 'POST', + headers: headers, + json: body, + timeout, + }; + + if (cancelToken) { + const abort = new AbortController(); + cancelToken.onCancellationRequested(() => { + // abort the request when the token is canceled + instantiationService.invokeFunction(telemetry, + 'networking.cancelRequest', + TelemetryData.createAndMarkAsIssued({ headerRequestId: requestId }) + ); + abort.abort(); + }); + // pass the controller abort signal to the request + request.signal = abort.signal; + } + + const requestPromise = fetcher.fetch(url, request).catch((reason: unknown) => { + if (isInterruptedNetworkError(reason)) { + // disconnect and retry the request once if the connection was reset + instantiationService.invokeFunction(telemetry, 'networking.disconnectAll'); + return fetcher.disconnectAll().then(() => { + return fetcher.fetch(url, request); + }); + } else { + throw reason; + } + }); + return requestPromise; +} + +export function isInterruptedNetworkError(error: unknown): boolean { + if (!(error instanceof Error)) { return false; } + if (error.message === 'ERR_HTTP2_GOAWAY_SESSION') { return true; } + if (!('code' in error)) { return false; } + return error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT' || error.code === 'ERR_HTTP2_INVALID_SESSION'; +} diff --git a/completions-sample-code/vscode-node/lib/src/networkingTypes.ts b/completions-sample-code/vscode-node/lib/src/networkingTypes.ts new file mode 100644 index 0000000..5fa5018 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/networkingTypes.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export { FetchOptions, Response } from '../../../../../platform/networking/common/fetcherService'; + +/** + * NETWORKING TYPES, INTERFACES AND ERROR CLASSES + * + * This module contains all networking-related types, interfaces, error classes and utilities. + */ + +class HttpTimeoutError extends Error { + constructor(message: string, cause?: unknown) { + super(message, { cause }); + this.name = 'HttpTimeoutError'; + } +} + +export function isAbortError(e: unknown): boolean { + if (!e || typeof e !== 'object') { + // Reject invalid errors + return false; + } + return ( + e instanceof HttpTimeoutError || + // internal Node.js AbortError, emitted by helix-fetch and electron net + ('name' in e && e.name === 'AbortError') || + // that same internal Node.js AbortError, but wrapped in a Helix FetchError + ('code' in e && e.code === 'ABORT_ERR') + ); +} + +export interface IAbortController { + readonly signal: IAbortSignal; + abort(): void; +} + +export interface IHeaders extends Iterable<[string, string]> { + append(name: string, value: string): void; + delete(name: string): void; + get(name: string): string | null; + has(name: string): boolean; + set(name: string, value: string): void; + + entries(): Iterator<[string, string]>; + keys(): Iterator<string>; + values(): Iterator<string>; + [Symbol.iterator](): Iterator<[string, string]>; +} + +export interface IAbortSignal extends Pick<EventTarget, 'addEventListener' | 'removeEventListener'> { + readonly aborted: boolean; +} + +export type ReqHeaders = { [key: string]: string }; diff --git a/completions-sample-code/vscode-node/lib/src/notificationSender.ts b/completions-sample-code/vscode-node/lib/src/notificationSender.ts new file mode 100644 index 0000000..ae429b9 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/notificationSender.ts @@ -0,0 +1,31 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { INotificationService } from '../../../../../platform/notification/common/notificationService'; +import { createServiceIdentifier } from '../../../../../util/common/services'; + +export interface ActionItem { + title: string; + [key: string]: string | boolean | object; +} + +export const ICompletionsNotificationSender = createServiceIdentifier<ICompletionsNotificationSender>('ICompletionsNotificationSender'); +export interface ICompletionsNotificationSender { + readonly _serviceBrand: undefined; + + showWarningMessage(message: string, ...actions: ActionItem[]): Promise<ActionItem | undefined>; +} + +export class ExtensionNotificationSender implements ICompletionsNotificationSender { + declare _serviceBrand: undefined; + + constructor(@INotificationService private readonly notificationService: INotificationService) { + } + + async showWarningMessage(message: string, ...actions: ActionItem[]): Promise<ActionItem | undefined> { + const response = await this.notificationService.showWarningMessage(message, ...actions.map(action => action.title)); + if (response === undefined) { return; } + return { title: response }; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/config.ts b/completions-sample-code/vscode-node/lib/src/openai/config.ts new file mode 100644 index 0000000..ab6e8cb --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/config.ts @@ -0,0 +1,35 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TokenizerName } from '../../../prompt/src/tokenization'; +import { TelemetryWithExp } from '../telemetry'; +import { CompletionHeaders } from './fetch'; +import { ICompletionsModelManagerService, ModelChoiceSourceTelemetryValue } from './model'; + +// Config methods + +export type EngineRequestInfo = { + headers: CompletionHeaders; + modelId: string; + engineChoiceSource: ModelChoiceSourceTelemetryValue; + tokenizer: TokenizerName; +}; + +export function getEngineRequestInfo( + accessor: ServicesAccessor, + telemetryData: TelemetryWithExp | undefined = undefined +): EngineRequestInfo { + const modelsManager = accessor.get(ICompletionsModelManagerService); + const modelRequestInfo = modelsManager.getCurrentModelRequestInfo(telemetryData); + const tokenizer = modelsManager.getTokenizerForModel(modelRequestInfo.modelId); + + return { + headers: modelRequestInfo.headers, + modelId: modelRequestInfo.modelId, + engineChoiceSource: modelRequestInfo.modelChoiceSource, + tokenizer, + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/fetch.fake.ts b/completions-sample-code/vscode-node/lib/src/openai/fetch.fake.ts new file mode 100644 index 0000000..3cbd070 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/fetch.fake.ts @@ -0,0 +1,188 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { CancellationToken } from 'vscode'; +import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; +import { getTokenizer } from '../../../prompt/src/tokenization'; +import { ICompletionsCopilotTokenManager } from '../auth/copilotTokenManager'; +import { Response } from '../networking'; +import { TelemetryData, TelemetryWithExp } from '../telemetry'; +import { + CompletionError, + CompletionParams, + CompletionResults, + FinishedCallback, + LiveOpenAIFetcher, + OpenAIFetcher, + PostOptions, + postProcessChoices, + SolutionDecision, + SpeculationFetchParams +} from './fetch'; +import { APIChoice } from './openai'; + +/** + * This module supports fake implementations of the completions returned by OpenAI, as well + * as injecting synthetic completions that would be hard to trigger directly but are useful + * for thoroughly testing the code that post-processes completions. + * + */ + +export function fakeAPIChoice( + headerRequestId: string, + choiceIndex: number, + completionText: string, + telemetryData: TelemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting() +): APIChoice { + const tokenizer = getTokenizer(); + + return { + completionText: completionText, + meanLogProb: 0.5, + meanAlternativeLogProb: 0.5, + numTokens: -1, + choiceIndex, + requestId: { + headerRequestId, + serverExperiments: 'dummy', + deploymentId: 'dummy', + gitHubRequestId: 'dummy', + completionId: 'dummy', + created: 0 + }, + telemetryData, + // This slightly convoluted way of getting the tokens as a string array is an + // alternative to exporting a way to do it directly from the tokenizer module. + tokens: tokenizer + .tokenize(completionText) + .map(token => tokenizer.detokenize([token])) + .concat(), + blockFinished: false, + clientCompletionId: generateUuid(), + finishReason: 'stop', + }; +} + +export function fakeAPIChoiceFromCompletion(completion: string): APIChoice { + return fakeAPIChoice(generateUuid(), 0, completion); +} + +export async function* fakeAPIChoices( + postOptions: PostOptions | undefined, + finishedCb: FinishedCallback, + completions: string[], + telemetryData?: TelemetryWithExp +): AsyncIterable<APIChoice> { + const fakeHeaderRequestId = generateUuid(); + let choiceIndex = 0; + for (let completion of completions) { + let stopOffset = -1; + if (postOptions?.stop !== undefined) { + for (const stopToken of postOptions.stop) { + const thisStopOffset = completion.indexOf(stopToken); + if (thisStopOffset !== -1 && (stopOffset === -1 || thisStopOffset < stopOffset)) { + stopOffset = thisStopOffset; + } + } + } + if (stopOffset !== -1) { + completion = completion.substring(0, stopOffset); + } + // This logic for using the finishedCb mirrors what happens in the live streamChoices function, + // but it doesn't try to stop reading the completion early as there's no point. + const finishOffset = asNumericOffset(await finishedCb(completion, { text: completion })); + if (finishOffset !== undefined) { + completion = completion.substring(0, finishOffset); + } + const choice = fakeAPIChoice(fakeHeaderRequestId, choiceIndex++, completion, telemetryData); + choice.blockFinished = finishOffset === undefined ? false : true; + yield choice; + } +} + +function asNumericOffset(result: SolutionDecision | number | undefined): number | undefined { + if (typeof result === 'number' || result === undefined) { + return result; + } + return result.finishOffset; +} + +function fakeResponse( + completions: string[], + finishedCb: FinishedCallback, + postOptions?: PostOptions, + telemetryData?: TelemetryWithExp +): Promise<CompletionResults> { + const choices = postProcessChoices(fakeAPIChoices(postOptions, finishedCb, completions, telemetryData)); + return Promise.resolve({ type: 'success', choices, getProcessingTime: () => 0 }); +} + +export class SyntheticCompletions extends OpenAIFetcher { + private _wasCalled = false; + + constructor( + private readonly _completions: string[], + @ICompletionsCopilotTokenManager private readonly copilotTokenManager: ICompletionsCopilotTokenManager, + ) { + super(); + } + + async fetchAndStreamCompletions( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancel?: CancellationToken, + teletryProperties?: { [key: string]: string } + ): Promise<CompletionResults | CompletionError> { + // check we have a valid token - ignore the result + void this.copilotTokenManager.getToken(); + if (cancel?.isCancellationRequested) { + return { type: 'canceled', reason: 'canceled during test' }; + } + + if (!this._wasCalled) { + this._wasCalled = true; + return fakeResponse(this._completions, finishedCb, params.postOptions, baseTelemetryData); + } else { + // In indentation mode, if the preview completion isn't enough to finish the completion, + // a second call will be made with the first prompt+preview completion as the prompt. + // As we've already returned everything we have, the second completion should be empty. + const emptyCompletions = this._completions.map(completion => ''); + return fakeResponse(emptyCompletions, finishedCb, params.postOptions, baseTelemetryData); + } + } + + async fetchAndStreamCompletions2( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancel?: CancellationToken + ): Promise<CompletionResults | CompletionError> { + return this.fetchAndStreamCompletions(params, baseTelemetryData, finishedCb, cancel); + } +} + + +export class ErrorReturningFetcher extends LiveOpenAIFetcher { + lastSpeculationParams?: CompletionParams | SpeculationFetchParams; + + private response: Response | 'not-sent' = 'not-sent'; + + setResponse(response: Response | 'not-sent') { + this.response = response; + } + + override fetchWithParameters( + endpoint: string, + params: CompletionParams, + _copilotToken: unknown, + telemetryData: TelemetryData, + cancel?: CancellationToken + ): Promise<Response | 'not-sent'> { + const response = this.response; + this.response = 'not-sent'; + return Promise.resolve(response); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/fetch.ts b/completions-sample-code/vscode-node/lib/src/openai/fetch.ts new file mode 100644 index 0000000..a31422c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/fetch.ts @@ -0,0 +1,1039 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { CopilotAnnotations, StreamCopilotAnnotations } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { IEnvService } from '../../../../../../platform/env/common/envService'; +import { Completion } from '../../../../../../platform/nesFetch/common/completionsAPI'; +import { Completions, ICompletionsFetchService } from '../../../../../../platform/nesFetch/common/completionsFetchService'; +import { ResponseStream } from '../../../../../../platform/nesFetch/common/responseStream'; +import { RequestId, getRequestId } from '../../../../../../platform/networking/common/fetch'; +import { IHeaders } from '../../../../../../platform/networking/common/fetcherService'; +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { assertNever } from '../../../../../../util/vs/base/common/assert'; +import { CancellationToken } from '../../../../../../util/vs/base/common/cancellation'; +import { StopWatch } from '../../../../../../util/vs/base/common/stopwatch'; +import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CancellationToken as ICancellationToken } from '../../../types/src'; +import { CopilotToken, ICompletionsCopilotTokenManager } from '../auth/copilotTokenManager'; +import { onCopilotToken } from '../auth/copilotTokenNotifier'; +import { apiVersion, editorVersionHeaders } from '../config'; +import { asyncIterableFilter, asyncIterableMap } from '../helpers/iterableHelpers'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { getEndpointUrl } from '../networkConfiguration'; +import { Response, isAbortError, isInterruptedNetworkError, postRequest } from '../networking'; +import { ICompletionsStatusReporter } from '../progress'; +import { Prompt } from '../prompt/prompt'; +import { MaybeRepoInfo, tryGetGitHubNWO } from '../prompt/repository'; +import { + TelemetryData, + TelemetryWithExp, + logEnginePrompt, + now, + telemetrizePromptLength, + telemetry, +} from '../telemetry'; +import { delay } from '../util/async'; +import { ICompletionsRuntimeModeService } from '../util/runtimeMode'; +import { getKey } from '../util/unknown'; +import { + APIChoice, + APIJsonData, + getMaxSolutionTokens, + getStops, + getTemperatureForSamples, + getTopP, +} from './openai'; +import { SSEProcessor, prepareSolutionForReturn } from './stream'; + +const logger = new Logger('fetchCompletions'); + +export enum CopilotUiKind { + GhostText = 'ghostText', + Panel = 'synthesize', // legacy value from the synthesize codelens +} + +type BaseFetchRequest = { + /** + * The prompt prefix to send to the model. Called `prompt` here for compatibility + * with the OpenAI API. + */ + prompt: string; +}; + +/** + * Request parameters other than the prompt, which will be included in the OAI + * API request. + */ +type CompletionFetchRequestFields = { + /** The prompt suffix to send to the model. */ + suffix: string; + /** Whether to stream back a response in SSE format. Always true: non streaming requests are not supported by this proxy */ + stream: true; + /** Maximum number of tokens the model should generate. */ + max_tokens: number; + /** How many parallel completions the model should generate (default 1). */ + n: number; + /** Non-negative temperature sampling parameter (default 1). */ + temperature: number; + /** Non-negative nucleus sampling parameter (defaults 1). */ + top_p: number; + /** Strings that will cause the model to stop generating text. */ + stop: string[]; + /** Number of alternative tokens to include logprob data for. */ + logprobs?: number; + /** Likelihood of specified tokens appearing in the completion. */ + logit_bias?: { [key: string]: number }; + + /** Copilot-only: NWO of repository, if any */ + nwo?: string; + /** + * Controls whether code citation annotations are included in the response + * stream for non-blocking requests. + */ + code_annotations?: boolean; +}; + +/** OAI API completion request, along with additional fields specific to Copilot. */ +export type CompletionRequest = BaseFetchRequest & + CompletionFetchRequestFields & { + /** Copilot-only: extra arguments for completion processing. */ + extra: Partial<CompletionRequestExtra>; + }; + +/** + * Completion request arguments that are Copilot-specific and don't exist in + * the OAI API. + */ +export declare interface CompletionRequestExtra { + /** The VSCode language ID for the file. */ + language: string; + /** + * If true, the proxy will trim completions to the current block/line based + * on the force_indent and/or next_indent values. + */ + trim_by_indentation?: boolean; + /** + * If set, will let the completion go on until a (non-continuation) line + * comes through with the given indentation level. + */ + force_indent?: number; + /** Number of leading space or tab characters in the next non-empty line. */ + next_indent?: number; + /** + * For testing only: A list of completions to be used instead of calling the + * model. The server will act as if the model returned these completions and + * postprocess them as it normally postprocesses model responses (i.e. + * filtering, trimming, etc.). + */ + test_completions?: string[]; + /** + The number of tokens (prefix) + */ + prompt_tokens: number; + /** + The number of tokens (suffix) + */ + suffix_tokens: number; + /** Additional context to send to the model. + * If this field is populated, then `prefix` will only contain the document prefix before the cursor.*/ + context?: string[]; +} + +export type PostOptions = Partial<CompletionFetchRequestFields>; + +// Request helpers + +function getProcessingTime(responseHeaders: IHeaders): number { + const reqIdStr = responseHeaders.get('openai-processing-ms'); + if (reqIdStr) { + return parseInt(reqIdStr, 10); + } + return 0; +} + +function uiKindToIntent(uiKind: CopilotUiKind): string | undefined { + switch (uiKind) { + case CopilotUiKind.GhostText: + return 'copilot-ghost'; + case CopilotUiKind.Panel: + return 'copilot-panel'; + } +} + +// Request methods + +export interface CopilotError { + type: string; + code: string; + message: string; + identifier: string; +} + +export interface CopilotConfirmation { + type: string; + title: string; + message: string; + confirmation: Record<string, unknown>; +} + +export interface CopilotReference { + type: string; + id: string; + data: Record<string, unknown>; +} + +export interface RequestDelta { + text: string; + index?: number; + requestId?: RequestId; + annotations?: CopilotAnnotations; + copilotErrors?: CopilotError[]; + copilotConfirmation?: CopilotConfirmation; + copilotReferences?: CopilotReference[]; + getAPIJsonData?: () => APIJsonData; + finished?: boolean; + telemetryData?: TelemetryWithExp; +} + +export interface SolutionDecision { + yieldSolution: boolean; + continueStreaming: boolean; + finishOffset?: number; +} + +type FinishedCallbackResult = + | Promise<SolutionDecision | number | undefined> + | SolutionDecision + | number + | undefined; + +/** + * Takes a (part of a) completion resolves to the offset of the end of the + * block, or undefined if the block is not yet finished. + */ +export interface FinishedCallback { + (text: string, delta: RequestDelta): FinishedCallbackResult; +} + +interface InternalFetchParams { + prompt: Prompt; + engineModelId: string; + uiKind: CopilotUiKind; + ourRequestId: string; + headers?: CompletionHeaders; +} + +/** + * Interface for the parameters passed to `fetchAndStreamCompletions` and `fetchWithParameters` wrappers, + * which then turn them into a `CompletionRequest` to be sent with `fetchWithInstrumentation`. + */ +export interface CompletionParams extends InternalFetchParams { + repoInfo: MaybeRepoInfo; + languageId: string; + count: number; + requestLogProbs?: boolean; + postOptions?: PostOptions; + extra: Partial<CompletionRequestExtra>; +} + +/** + * Interface for the parameters passed to `fetchSpeculationWithParameters`, + * which then turns them into a `SpeculationCompletionRequest` object to be sent with `fetchWithInstrumentation`. + */ +export interface SpeculationFetchParams extends InternalFetchParams { + speculation: string; + stops: string[] | null; +} + +export const ICompletionsOpenAIFetcherService = createServiceIdentifier<ICompletionsOpenAIFetcherService>('ICompletionsOpenAIFetcherService'); +export interface ICompletionsOpenAIFetcherService { + readonly _serviceBrand: undefined; + fetchAndStreamCompletions( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancellationToken?: ICancellationToken + ): Promise<CompletionResults | CompletionError>; + fetchAndStreamCompletions2( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancellationToken?: ICancellationToken + ): Promise<CompletionResults | CompletionError>; +} + +/** An interface to abstract away the network request to OpenAI, allowing for + * fake or mock implementations. It's deliberately injected relatively high + * in the call stack to avoid having to reconstruct some of the lower-level details + * of the OpenAI API. + */ +export abstract class OpenAIFetcher implements ICompletionsOpenAIFetcherService { + declare _serviceBrand: undefined; + /** + * Sends a request to the code completion endpoint. + */ + abstract fetchAndStreamCompletions( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancellationToken?: ICancellationToken + ): Promise<CompletionResults | CompletionError>; + abstract fetchAndStreamCompletions2( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancellationToken?: ICancellationToken + ): Promise<CompletionResults | CompletionError>; +} + +export interface CompletionResults { + type: 'success'; + choices: AsyncIterable<APIChoice>; + getProcessingTime(): number; +} + +export type CompletionError = { type: 'failed'; reason: string } | { type: 'canceled'; reason: string }; + +export type CompletionHeaders = { + /** For speculation only**/ + Host?: string; + Connection?: string; + 'X-Copilot-Async'?: string; + 'X-Copilot-Speculative'?: string; +}; + +function getProxyEngineUrl(accessor: ServicesAccessor, token: CopilotToken, modelId: string, endpoint: string): string { + return getEndpointUrl(accessor, token, 'proxy', 'v1/engines', modelId, endpoint); +} + +export function sanitizeRequestOptionTelemetry( + request: Partial<CompletionRequest>, + telemetryData: TelemetryWithExp, + topLevelKeys: string[], // top-level properties to exclude from standard telemetry + extraKeys?: (keyof CompletionRequestExtra)[] // keys under the `extra` property to exclude from standard telemetry +): void { + for (const [key, value] of Object.entries(request)) { + if (topLevelKeys.includes(key)) { + continue; + } + + let valueToLog = value as unknown; + + if (key === 'extra' && extraKeys) { + const extra = { ...(valueToLog as CompletionRequestExtra) }; + for (const extraKey of extraKeys) { + delete extra[extraKey]; + } + valueToLog = extra; + } + + telemetryData.properties[`request.option.${key}`] = JSON.stringify(valueToLog) ?? 'undefined'; + } +} + +async function fetchWithInstrumentation( + accessor: ServicesAccessor, + prompt: Prompt, + engineModelId: string, + endpoint: string, + ourRequestId: string, + request: Record<string, unknown>, + copilotToken: CopilotToken, + uiKind: CopilotUiKind, + telemetryExp: TelemetryWithExp, + cancel?: ICancellationToken, + headers?: CompletionHeaders +): Promise<Response> { + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const statusReporter = accessor.get(ICompletionsStatusReporter); + const uri = instantiationService.invokeFunction(getProxyEngineUrl, copilotToken, engineModelId, endpoint); + + const telemetryData = telemetryExp.extendedBy( + { + endpoint: endpoint, + engineName: engineModelId, + uiKind: uiKind, + }, + telemetrizePromptLength(prompt) + ); + + // Skip prompt info (PII) + sanitizeRequestOptionTelemetry(request, telemetryData, ['prompt', 'suffix'], ['context']); + + // The request ID we are passed in is sent in the request to the proxy, and included in our pre-request telemetry. + // We hope (but do not rely on) that the model will use the same ID in the response, allowing us to correlate + // the request and response. + telemetryData.properties['headerRequestId'] = ourRequestId; + + instantiationService.invokeFunction(telemetry, 'request.sent', telemetryData); + + const requestStart = now(); + const intent = uiKindToIntent(uiKind); + + // Wrap the Promise with success/error callbacks so we can log/measure it + return instantiationService.invokeFunction(postRequest, uri, copilotToken.token, intent, ourRequestId, request, cancel, headers) + .then(response => { + // This ID is hopefully the one the same as ourRequestId, but it is not guaranteed. + // If they are different then we will override the original one we set in telemetryData above. + const modelRequestId = getRequestId(response.headers); + telemetryData.extendWithRequestId(modelRequestId); + + // TODO: Add response length (requires parsing) + const totalTimeMs = now() - requestStart; + telemetryData.measurements.totalTimeMs = totalTimeMs; + + logger.info( + logTarget, + `Request ${ourRequestId} at <${uri}> finished with ${response.status} status after ${totalTimeMs}ms` + ); + telemetryData.properties.status = String(response.status); + logger.debug(logTarget, 'request.response properties', telemetryData.properties); + logger.debug(logTarget, 'request.response measurements', telemetryData.measurements); + + logger.debug(logTarget, 'prompt:', prompt); + + instantiationService.invokeFunction(telemetry, 'request.response', telemetryData); + + return response; + }) + .catch((error: unknown) => { + if (isAbortError(error)) { + // If we cancelled a network request, we want to log a `request.cancel` instead of `request.error` + instantiationService.invokeFunction(telemetry, 'request.cancel', telemetryData); + throw error; + } + statusReporter.setWarning(getKey(error, 'message') ?? ''); + const warningTelemetry = telemetryData.extendedBy({ error: 'Network exception' }); + instantiationService.invokeFunction(telemetry, 'request.shownWarning', warningTelemetry); + + telemetryData.properties.message = String(getKey(error, 'name') ?? ''); + telemetryData.properties.code = String(getKey(error, 'code') ?? ''); + telemetryData.properties.errno = String(getKey(error, 'errno') ?? ''); + telemetryData.properties.type = String(getKey(error, 'type') ?? ''); + + const totalTimeMs = now() - requestStart; + telemetryData.measurements.totalTimeMs = totalTimeMs; + + logger.info( + logTarget, + `Request ${ourRequestId} at <${uri}> rejected with ${String(error)} after ${totalTimeMs}ms` + ); + logger.debug(logTarget, 'request.error properties', telemetryData.properties); + logger.debug(logTarget, 'request.error measurements', telemetryData.measurements); + + instantiationService.invokeFunction(telemetry, 'request.error', telemetryData); + + throw error; + }) + .finally(() => { + instantiationService.invokeFunction(logEnginePrompt, prompt, telemetryData); + }); +} + +export function postProcessChoices(choices: AsyncIterable<APIChoice>) { + return asyncIterableFilter(choices, choice => choice.completionText.trim().length > 0); +} + +export const CMDQuotaExceeded = 'github.copilot.completions.quotaExceeded'; + +export class LiveOpenAIFetcher extends OpenAIFetcher { + #disabledReason: string | undefined; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsRuntimeModeService private readonly runtimeModeService: ICompletionsRuntimeModeService, + @ICompletionsLogTargetService private readonly logTargetService: ICompletionsLogTargetService, + @ICompletionsCopilotTokenManager private readonly copilotTokenManager: ICompletionsCopilotTokenManager, + @ICompletionsStatusReporter private readonly statusReporter: ICompletionsStatusReporter, + @IAuthenticationService private readonly authenticationService: IAuthenticationService, + @ICompletionsFetchService private readonly fetchService: ICompletionsFetchService, + // @ICompletionsLogTargetService private readonly logTarget: ICompletionsLogTargetService, + @IEnvService private readonly envService: IEnvService, + ) { + super(); + } + + async fetchAndStreamCompletions( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancel?: ICancellationToken + ): Promise<CompletionResults | CompletionError> { + if (this.#disabledReason) { + return { type: 'canceled', reason: this.#disabledReason }; + } + const endpoint = 'completions'; + const copilotToken = this.copilotTokenManager.token ?? await this.copilotTokenManager.getToken(); + const response = await this.fetchWithParameters(endpoint, params, copilotToken, baseTelemetryData, cancel); + if (response === 'not-sent') { + return { type: 'canceled', reason: 'before fetch request' }; + } + if (cancel?.isCancellationRequested) { + try { + // Destroy the stream so that the server is hopefully notified we don't want any more data + // and can cancel/forget about the request itself. + await response.body.destroy(); + } catch (e) { + this.instantiationService.invokeFunction(acc => logger.exception(acc, e, `Error destroying stream`)); + } + return { type: 'canceled', reason: 'after fetch request' }; + } + + if (response.status !== 200) { + const telemetryData = this.createTelemetryData(endpoint, params); + return this.handleError(this.statusReporter, telemetryData, response, copilotToken); + } + const processor = await this.instantiationService.invokeFunction(SSEProcessor.create, params.count, response, baseTelemetryData, [], cancel); + const finishedCompletions = processor.processSSE(finishedCb); + const choices = asyncIterableMap(finishedCompletions, solution => + this.instantiationService.invokeFunction(prepareSolutionForReturn, solution, baseTelemetryData) + ); + return { + type: 'success', + choices: postProcessChoices(choices), + getProcessingTime: () => getProcessingTime(response.headers), + }; + } + + async fetchAndStreamCompletions2( + params: CompletionParams, + baseTelemetryData: TelemetryWithExp, + finishedCb: FinishedCallback, + cancel: CancellationToken + ): Promise<CompletionResults | CompletionError> { + if (this.#disabledReason) { + return { type: 'canceled', reason: this.#disabledReason }; + } + const endpoint = 'completions'; + const copilotToken = this.copilotTokenManager.token ?? await this.copilotTokenManager.getToken(); + + // fetchWithParameters - start + const request: CompletionRequest = { + prompt: params.prompt.prefix, + suffix: params.prompt.suffix, + max_tokens: getMaxSolutionTokens(), + temperature: getTemperatureForSamples(this.runtimeModeService, params.count), + top_p: getTopP(), + n: params.count, + stop: getStops(params.languageId), + stream: true, // Always true: non streaming requests are not supported by this proxy + extra: params.extra, + } satisfies CompletionRequest; + + { + + if (params.requestLogProbs) { + request.logprobs = 2; // Request that logprobs of 2 tokens (i.e. including the best alternative) be returned + } + + const githubNWO = tryGetGitHubNWO(params.repoInfo); + if (githubNWO !== undefined) { + request.nwo = githubNWO; + } + + if (params.postOptions) { + Object.assign(request, params.postOptions); + } + + if (params.prompt.context && params.prompt.context.length > 0) { + request.extra.context = params.prompt.context; + } + + // Give a final opportunity to cancel the request before we send the request + // This await line is necessary to allow the tests in extension/src/openai.test.ts to pass + await delay(0); + if (cancel?.isCancellationRequested) { + // return 'not-sent'; + return { type: 'canceled', reason: 'before fetch request' }; + } + } + // fetchWithParameters - end + + // fetchWithInstrumentation - start + { + const prompt = params.prompt; + const engineModelId = params.engineModelId; + const ourRequestId = params.ourRequestId; + const telemetryExp = baseTelemetryData; + const uiKind = params.uiKind; + const headers = params.headers; + // const logTarget = this.logTarget; + + const uri = this.instantiationService.invokeFunction(getProxyEngineUrl, copilotToken, engineModelId, endpoint); + + const telemetryData = telemetryExp.extendedBy( + { + endpoint: endpoint, + engineName: engineModelId, + uiKind: uiKind, + }, + telemetrizePromptLength(prompt) + ); + + // Skip prompt info (PII) + sanitizeRequestOptionTelemetry(request, telemetryData, ['prompt', 'suffix'], ['context']); + + // The request ID we are passed in is sent in the request to the proxy, and included in our pre-request telemetry. + // We hope (but do not rely on) that the model will use the same ID in the response, allowing us to correlate + // the request and response. + telemetryData.properties['headerRequestId'] = ourRequestId; + + this.instantiationService.invokeFunction(telemetry, 'request.sent', telemetryData); + + // const requestStart = now(); + const intent = uiKindToIntent(uiKind); + + // Wrap the Promise with success/error callbacks so we can log/measure it + // return this.instantiationService.invokeFunction(postRequest, uri, copilotToken.token, intent, ourRequestId, request, cancel, headers) + + let fullHeaders: Record<string, string>; + + // postRequest - start + { + fullHeaders = { + ...headers, + ...this.instantiationService.invokeFunction(editorVersionHeaders), + }; + + // If we call byok endpoint, no need to add these headers + // if (modelProviderName === undefined) { + fullHeaders['Openai-Organization'] = 'github-copilot'; + fullHeaders['X-Request-Id'] = ourRequestId; + fullHeaders['VScode-SessionId'] = this.envService.sessionId; + fullHeaders['VScode-MachineId'] = this.envService.machineId; + fullHeaders['X-GitHub-Api-Version'] = apiVersion; + // } + + if (intent) { + fullHeaders['OpenAI-Intent'] = intent; + } + } + // postRequest - end + + const requestSw = new StopWatch(); + const res = await this.fetchService.fetch( + uri, + copilotToken.token, + request, + ourRequestId, + cancel, + fullHeaders, + ).then(response => { + if (response.isError() && response.err instanceof Completions.Unexpected && isInterruptedNetworkError(response.err.error)) { + // disconnect and retry the request once if the connection was reset + this.instantiationService.invokeFunction(telemetry, 'networking.disconnectAll'); + return this.fetchService.disconnectAll().then(() => { + return this.fetchService.fetch( + uri, + copilotToken.token, + request, + ourRequestId, + cancel, + fullHeaders, + ); + }); + } else { + return response; + } + }); + + // .finally from fetchWithInstrumentation + this.instantiationService.invokeFunction(logEnginePrompt, prompt, telemetryData); + + if (res.isError()) { + + const err = res.err; + + if (err instanceof Completions.RequestCancelled) { + // abort the request when the token is canceled + this.instantiationService.invokeFunction(telemetry, + 'networking.cancelRequest', + TelemetryData.createAndMarkAsIssued({ headerRequestId: ourRequestId }) + ); + return { type: 'canceled', reason: 'during fetch request' }; + } else if (err instanceof Completions.UnsuccessfulResponse) { + const telemetryData = this.createTelemetryData(endpoint, params); // FIXME + return this.handleError(this.statusReporter, telemetryData, { + status: err.status, + text: err.text, + headers: err.headers, + }, copilotToken); + } else if (err instanceof Completions.Unexpected) { + + const error = err.error; + + // .catch from fetchWithInstrumentation + if (isAbortError(error)) { + // If we cancelled a network request, we want to log a `request.cancel` instead of `request.error` + this.instantiationService.invokeFunction(telemetry, 'request.cancel', telemetryData); + throw error; + } + this.statusReporter.setWarning(getKey(error, 'message') ?? ''); + const warningTelemetry = telemetryData.extendedBy({ error: 'Network exception' }); + this.instantiationService.invokeFunction(telemetry, 'request.shownWarning', warningTelemetry); + + telemetryData.properties.message = String(getKey(error, 'name') ?? ''); + telemetryData.properties.code = String(getKey(error, 'code') ?? ''); + telemetryData.properties.errno = String(getKey(error, 'errno') ?? ''); + telemetryData.properties.type = String(getKey(error, 'type') ?? ''); + + const totalTimeMs = requestSw.elapsed(); + telemetryData.measurements.totalTimeMs = totalTimeMs; + + logger.info( + this.logTargetService, + `Request ${ourRequestId} at <${uri}> rejected with ${String(error)} after ${totalTimeMs}ms` + ); + logger.debug(this.logTargetService, 'request.error properties', telemetryData.properties); + logger.debug(this.logTargetService, 'request.error measurements', telemetryData.measurements); + + this.instantiationService.invokeFunction(telemetry, 'request.error', telemetryData); + + throw error; + } else { + assertNever(err); + } + } + + const responseStream = res.val; + + // .then from fetchWithInstrumentation + { + // This ID is hopefully the one the same as ourRequestId, but it is not guaranteed. + // If they are different then we will override the original one we set in telemetryData above. + const modelRequestId = responseStream.requestId; + telemetryData.extendWithRequestId(modelRequestId); + + // TODO: Add response length (requires parsing) + const totalTimeMs = requestSw.elapsed(); + telemetryData.measurements.totalTimeMs = totalTimeMs; + + const responseStatus = 200; // because otherwise it wouldn't be here + logger.info( + this.logTargetService, + `Request ${ourRequestId} at <${uri}> finished with ${responseStatus} status after ${totalTimeMs}ms` + ); + telemetryData.properties.status = String(responseStatus); + logger.debug(this.logTargetService, 'request.response properties', telemetryData.properties); + logger.debug(this.logTargetService, 'request.response measurements', telemetryData.measurements); + + logger.debug(this.logTargetService, 'prompt:', prompt); + + this.instantiationService.invokeFunction(telemetry, 'request.response', telemetryData); + + } + + if (cancel.isCancellationRequested) { + try { + // Destroy the stream so that the server is hopefully notified we don't want any more data + // and can cancel/forget about the request itself. + await responseStream.destroy(); + } catch (e) { + this.instantiationService.invokeFunction(acc => logger.exception(acc, e, `Error destroying stream`)); + } + return { type: 'canceled', reason: 'after fetch request' }; + } + + const choices = LiveOpenAIFetcher.convertStreamToApiChoices(responseStream, finishedCb, baseTelemetryData); + + return { + type: 'success', + choices, + getProcessingTime: () => getProcessingTime(responseStream.headers), + }; + } + } + + private createTelemetryData(endpoint: string, params: CompletionParams | SpeculationFetchParams) { + return TelemetryData.createAndMarkAsIssued({ + endpoint: endpoint, + engineName: params.engineModelId, + uiKind: params.uiKind, + headerRequestId: params.ourRequestId, + }); + } + + protected async fetchWithParameters( + endpoint: string, + params: CompletionParams, + copilotToken: CopilotToken, + baseTelemetryData: TelemetryWithExp, + cancel?: ICancellationToken + ): Promise<Response | 'not-sent'> { + + const request: CompletionRequest = { + prompt: params.prompt.prefix, + suffix: params.prompt.suffix, + max_tokens: getMaxSolutionTokens(), + temperature: getTemperatureForSamples(this.runtimeModeService, params.count), + top_p: getTopP(), + n: params.count, + stop: getStops(params.languageId), + stream: true, // Always true: non streaming requests are not supported by this proxy + extra: params.extra, + }; + + if (params.requestLogProbs) { + request.logprobs = 2; // Request that logprobs of 2 tokens (i.e. including the best alternative) be returned + } + + const githubNWO = tryGetGitHubNWO(params.repoInfo); + if (githubNWO !== undefined) { + request.nwo = githubNWO; + } + + if (params.postOptions) { + Object.assign(request, params.postOptions); + } + + if (params.prompt.context && params.prompt.context.length > 0) { + request.extra.context = params.prompt.context; + } + + // Give a final opportunity to cancel the request before we send the request + // This await line is necessary to allow the tests in extension/src/openai.test.ts to pass + await delay(0); + if (cancel?.isCancellationRequested) { + return 'not-sent'; + } + + const response = await this.instantiationService.invokeFunction( + fetchWithInstrumentation, + params.prompt, + params.engineModelId, + endpoint, + params.ourRequestId, + request, + copilotToken, + params.uiKind, + baseTelemetryData, + cancel, + params.headers + ); + return response; + } + + /** + * @remarks exposed only for testing. + */ + public static async *convertStreamToApiChoices(resp: ResponseStream, finishedCb: FinishedCallback, baseTelemetryData: TelemetryWithExp): AsyncIterable<APIChoice> { + + const createAPIChoice = ( + choiceIndex: number, + completionText: string, + finishReason: string, + accumulator: CompletionAccumulator, + blockFinished: boolean + ): APIChoice => ({ + choiceIndex, + completionText, + requestId: resp.requestId, + finishReason, + tokens: accumulator.chunks, + numTokens: accumulator.chunks.length, + blockFinished, + telemetryData: baseTelemetryData, + clientCompletionId: generateUuid(), + meanLogProb: undefined, + meanAlternativeLogProb: undefined, + }); + + const completions: { accumulator: CompletionAccumulator; isFinished: boolean }[] = []; + + for await (const chunk of resp.stream) { + for (let i = 0; i < chunk.choices.length; i++) { + const chunkIdx = chunk.choices[i].index; + let completion = completions[chunkIdx]; + if (completion === undefined) { + completion = { accumulator: new CompletionAccumulator(), isFinished: false }; + completions[chunkIdx] = completion; + } else if (completion.isFinished) { + // already finished, skip + continue; + } + const choice = chunk.choices[i]; + completion.accumulator.append(choice); + + // finish_reason determines whether the completion is finished by the LLM + const hasFinishReason = !!(chunk.choices[i].finish_reason); + + // call finishedCb to determine whether the completion is finished by the client + const solutionDecision = await finishedCb(completion.accumulator.responseSoFar, { + index: chunkIdx, + text: completion.accumulator.responseSoFar, + finished: hasFinishReason, + requestId: resp.requestId, + telemetryData: baseTelemetryData, + annotations: completion.accumulator.annotations, + getAPIJsonData: () => ({ + text: completion.accumulator.responseSoFar, + tokens: completion.accumulator.chunks, + finish_reason: completion.accumulator.finishReason ?? 'stop', // @ulugbekna: logic to determine if last completion was accepted uses finish reason, so changing this `?? 'stop'` will change behavior of multiline completions + copilot_annotations: completion.accumulator.annotations.current, + } satisfies APIJsonData), + } satisfies RequestDelta); + + // handle all fields of finishedCb + if (hasFinishReason || + (solutionDecision !== undefined && + (typeof solutionDecision === 'number' || solutionDecision.yieldSolution)) + ) { + // mark as finished + const isFinished = hasFinishReason || typeof solutionDecision === 'number' || (solutionDecision !== undefined && !solutionDecision.continueStreaming); + completion.isFinished = isFinished; + + const finishReason = chunk.choices[i].finish_reason; + if (finishReason) { + completion.accumulator.finishReason = finishReason; + } + + const finishOffset = typeof solutionDecision === 'number' + ? solutionDecision + : (solutionDecision && solutionDecision.finishOffset !== undefined + ? solutionDecision.finishOffset + : undefined); + const completionText = finishOffset === undefined + ? completion.accumulator.responseSoFar + : completion.accumulator.responseSoFar.slice(0, finishOffset); + + const choice = createAPIChoice( + chunkIdx, + completionText, + completion.accumulator.finishReason ?? 'stop', + completion.accumulator, + finishOffset !== undefined, + ); + yield choice; + } + } + } + + // in case stream ends but some completions are not finished yet + for (const [chunkIdx, completion] of completions.entries()) { + if (!completion.isFinished) { + const choice = createAPIChoice( + chunkIdx, + completion.accumulator.responseSoFar, + 'DONE', // @ulugbekna: should match original ghost-text fetcher behavior + completion.accumulator, + false + ); + yield choice; + } + } + } + + async handleError( + statusReporter: ICompletionsStatusReporter, + telemetryData: TelemetryData, + response: { status: number; text(): Promise<string>; headers: IHeaders }, + copilotToken: CopilotToken + ): Promise<CompletionError> { + const text = await response.text(); + if (response.status === 402) { + this.#disabledReason = 'monthly free code completions exhausted'; + const message = 'Completions limit reached'; + statusReporter.setError(message, { + command: CMDQuotaExceeded, + title: 'Learn More', + }); + const event = onCopilotToken(this.authenticationService, t => { + this.#disabledReason = undefined; + if (!t.isCompletionsQuotaExceeded) { + statusReporter.forceNormal(); + event.dispose(); + } + }); + return { type: 'failed', reason: this.#disabledReason }; + } + if (response.status === 466) { + statusReporter.setError(text); + logger.info(this.logTargetService, text); + return { type: 'failed', reason: `client not supported: ${text}` }; + } + if (isClientError(response) && !response.headers.get('x-github-request-id')) { + const message = `Last response was a ${response.status} error and does not appear to originate from GitHub. Is a proxy or firewall intercepting this request? https://gh.io/copilot-firewall`; + logger.error(this.logTargetService, message); + statusReporter.setWarning(message); + telemetryData.properties.error = `Response status was ${response.status} with no x-github-request-id header`; + } else if (isClientError(response)) { + logger.warn(this.logTargetService, `Response status was ${response.status}:`, text); + statusReporter.setWarning(`Last response was a ${response.status} error: ${text}`); + telemetryData.properties.error = `Response status was ${response.status}: ${text}`; + } else { + statusReporter.setWarning(`Last response was a ${response.status} error`); + telemetryData.properties.error = `Response status was ${response.status}`; + } + telemetryData.properties.status = String(response.status); + this.instantiationService.invokeFunction(telemetry, 'request.shownWarning', telemetryData); + // check for 4xx responses which will point to a forbidden + if (response.status === 401 || response.status === 403) { + // Token has expired or invalid, fetch a new one on next request + // TODO(drifkin): these actions should probably happen in vsc specific code + this.copilotTokenManager.resetToken(response.status); + return { type: 'failed', reason: `token expired or invalid: ${response.status}` }; + } + if (response.status === 429) { + const rateLimitSeconds = 10; + setTimeout(() => { + this.#disabledReason = undefined; + }, rateLimitSeconds * 1000); + this.#disabledReason = 'rate limited'; + logger.warn(this.logTargetService, `Rate limited by server. Denying completions for the next ${rateLimitSeconds} seconds.`); + return { type: 'failed', reason: this.#disabledReason }; + } + if (response.status === 499) { + logger.info(this.logTargetService, 'Cancelled by server'); + return { type: 'failed', reason: 'canceled by server' }; + } + logger.error(this.logTargetService, 'Unhandled status from server:', response.status, text); + return { type: 'failed', reason: `unhandled status from server: ${response.status} ${text}` }; + } +} + +function isClientError(response: { status: number }): boolean { + return response.status >= 400 && response.status < 500; +} + +class CompletionAccumulator { + + private _chunks: string[] = []; + /** concatenated version of {_chunks} */ + private _responseSoFar: string = ''; + + private _finishReason: string | null = null; + + public readonly annotations: CopilotAnnotations = new StreamCopilotAnnotations(); + + public get responseSoFar(): string { + return this._responseSoFar; + } + + public get chunks(): readonly string[] { + return this._chunks; + } + + public set finishReason(value: string) { + this._finishReason = value; + } + public get finishReason(): string | null { + return this._finishReason; + } + + public append(choice: Completion.Choice): void { + const chunk = choice.text; + if (chunk) { + this._chunks.push(chunk); + this._responseSoFar = this._responseSoFar + chunk; + } + + if (choice.copilot_annotations) { + this.annotations.update(choice.copilot_annotations); + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/model.ts b/completions-sample-code/vscode-node/lib/src/openai/model.ts new file mode 100644 index 0000000..0d31472 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/model.ts @@ -0,0 +1,174 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { ICompletionModelInformation, IEndpointProvider } from '../../../../../../platform/endpoint/common/endpointProvider'; +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { Disposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TokenizerName } from '../../../prompt/src/tokenization'; +import { onCopilotToken } from '../auth/copilotTokenNotifier'; +import { ConfigKey, getConfig } from '../config'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { TelemetryWithExp } from '../telemetry'; +import { CompletionHeaders } from './fetch'; + +export const ICompletionsModelManagerService = createServiceIdentifier<ICompletionsModelManagerService>('ICompletionsModelManagerService'); +export interface ICompletionsModelManagerService { + readonly _serviceBrand: undefined; + getGenericCompletionModels(): ModelItem[]; + getDefaultModelId(): string; + getTokenizerForModel(modelId: string): TokenizerName; + getCurrentModelRequestInfo(featureSettings?: TelemetryWithExp): ModelRequestInfo; +} + +const FallbackModelId = 'gpt-41-copilot'; +export class AvailableModelsManager extends Disposable implements ICompletionsModelManagerService { + declare _serviceBrand: undefined; + fetchedModelData: ICompletionModelInformation[] = []; + customModels: string[] = []; + editorPreviewFeaturesDisabled: boolean = false; + + constructor( + shouldFetch: boolean = true, + @IInstantiationService private readonly _instantiationService: IInstantiationService, + @ICompletionsFeaturesService private readonly _featuresService: ICompletionsFeaturesService, + @IEndpointProvider private readonly _endpointProvider: IEndpointProvider, + @IAuthenticationService authenticationService: IAuthenticationService, + ) { + super(); + + if (shouldFetch) { + this._register(onCopilotToken(authenticationService, () => this.refreshAvailableModels())); + } + } + + // This will get its initial call after the initial token got fetched + private async refreshAvailableModels(): Promise<void> { + await this.refreshModels(); + } + + /** + * Returns the default model, determined by the order returned from the API + * Note: this does NOT fetch models to avoid side effects + */ + getDefaultModelId(): string { + if (this.fetchedModelData) { + const fetchedDefaultModel = AvailableModelsManager.filterCompletionModels( + this.fetchedModelData, + this.editorPreviewFeaturesDisabled + )[0]; + + if (fetchedDefaultModel) { + return fetchedDefaultModel.id; + } + } + + return FallbackModelId; + } + + async refreshModels(): Promise<void> { + const fetchedData = await this._endpointProvider.getAllCompletionModels(true); + if (fetchedData) { + this.fetchedModelData = fetchedData; + } + } + + /** + * Returns a list of models that are available for generic completions. + * Calls to CAPI to retrieve the list. + */ + getGenericCompletionModels(): ModelItem[] { + const filteredResult = AvailableModelsManager.filterCompletionModels( + this.fetchedModelData, + this.editorPreviewFeaturesDisabled + ); + + return AvailableModelsManager.mapCompletionModels(filteredResult); + } + + getTokenizerForModel(modelId: string): TokenizerName { + const modelItems = this.getGenericCompletionModels(); + const modelItem = modelItems.find(item => item.modelId === modelId); + if (modelItem) { + return modelItem.tokenizer as TokenizerName; + } + // The tokenizer the default model uses + return TokenizerName.o200k; + } + + static filterCompletionModels(data: ICompletionModelInformation[], editorPreviewFeaturesDisabled: boolean): ICompletionModelInformation[] { + return data + .filter(item => item.capabilities.type === 'completion') + .filter(item => !editorPreviewFeaturesDisabled || item.preview === false || item.preview === undefined); + } + + static filterModelsWithEditorPreviewFeatures( + data: ICompletionModelInformation[], + editorPreviewFeaturesDisabled: boolean + ): ICompletionModelInformation[] { + return data.filter( + item => !editorPreviewFeaturesDisabled || item.preview === false || item.preview === undefined + ); + } + + static mapCompletionModels(data: ICompletionModelInformation[]): ModelItem[] { + return data.map(item => ({ + modelId: item.id, + label: item.name, + preview: !!item.preview, + tokenizer: item.capabilities.tokenizer, + })); + } + + getCurrentModelRequestInfo(featureSettings: TelemetryWithExp | undefined = undefined): ModelRequestInfo { + const defaultModelId = this.getDefaultModelId(); + + const debugOverride = + this._instantiationService.invokeFunction(getConfig<string>, ConfigKey.DebugOverrideEngine) || + this._instantiationService.invokeFunction(getConfig<string>, ConfigKey.DebugOverrideEngineLegacy); + + if (debugOverride) { + return new ModelRequestInfo(debugOverride, 'override'); + } + + const customEngine = featureSettings ? this._featuresService.customEngine(featureSettings) : ''; + if (customEngine) { + return new ModelRequestInfo(customEngine, 'exp'); + } + + if (this.customModels.length > 0) { + return new ModelRequestInfo(this.customModels[0], 'custommodel'); + } + + return new ModelRequestInfo(defaultModelId, 'default'); + } +} + +export interface ModelItem { + modelId: string; + label: string; + preview: boolean; + tokenizer: string; +} + +export type ModelChoiceSourceTelemetryValue = + | 'override' + | 'modelpicker' + | 'exp' + | 'default' + | 'custommodel' + | 'prerelease'; + +class ModelRequestInfo { + constructor( + readonly modelId: string, + readonly modelChoiceSource: ModelChoiceSourceTelemetryValue + ) { } + + get headers(): CompletionHeaders { + return {}; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/openai.ts b/completions-sample-code/vscode-node/lib/src/openai/openai.ts new file mode 100644 index 0000000..d313b4b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/openai.ts @@ -0,0 +1,175 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { RequestId } from '../../../../../../platform/networking/common/fetch'; +import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { DEFAULT_MAX_COMPLETION_LENGTH } from '../../../prompt/src/prompt'; +import { logger } from '../logger'; +import { TelemetryWithExp, logEngineCompletion } from '../telemetry'; +import { ICompletionsRuntimeModeService } from '../util/runtimeMode'; + +export { FinishedCallback } from './fetch'; + +export interface APIChoice { + completionText: string; + meanLogProb: number | undefined; + meanAlternativeLogProb: number | undefined; + choiceIndex: number; + requestId: RequestId; + tokens: readonly string[]; + numTokens: number; + blockFinished: boolean; // Whether the block completion was determined to be finished + telemetryData: TelemetryWithExp; // optional telemetry data providing background + copilotAnnotations?: CopilotNamedAnnotationList; // optional annotations from the proxy + clientCompletionId: string; // Unique identifier for the completion created in the client + finishReason: string; // Reason the API used to describe why the stream of chunks finished. + generatedChoiceIndex?: number; // when a completion is split into multiple choices, the index of the split choice +} + +/** How the logprobs field looks in the OpenAI API chunks. */ +export interface APILogprobs { + text_offset: number[]; + token_logprobs: number[]; + top_logprobs?: { [key: string]: number }[]; + tokens: string[]; +} + +export interface APIJsonData { + text: string; + /* Joining this together produces `text`, due to the way the proxy works. */ + tokens: readonly string[]; + /* These are only generated in certain situations. */ + logprobs?: APILogprobs; + /* Copilot-specific annotations returned by the proxy. */ + copilot_annotations?: CopilotNamedAnnotationList; + /* Reason the proxy returned for why the stream of chunks ended. */ + finish_reason: string; // Reason the API used to describe why the stream of chunks finished. +} + +export function convertToAPIChoice( + accessor: ServicesAccessor, + completionText: string, + jsonData: APIJsonData, + choiceIndex: number, + requestId: RequestId, + blockFinished: boolean, + telemetryData: TelemetryWithExp +): APIChoice { + logEngineCompletion(accessor, completionText, jsonData, requestId, choiceIndex); + + // NOTE: It's possible that the completion text we care about is not exactly jsonData.text but a prefix, + // so we pass it down directly. + return { + // NOTE: This does not contain stop tokens necessarily + completionText: completionText, + meanLogProb: calculateMeanLogProb(accessor, jsonData), + meanAlternativeLogProb: calculateMeanAlternativeLogProb(accessor, jsonData), + choiceIndex: choiceIndex, + requestId: requestId, + blockFinished: blockFinished, + tokens: jsonData.tokens, + numTokens: jsonData.tokens.length, + telemetryData: telemetryData, + copilotAnnotations: jsonData.copilot_annotations, + clientCompletionId: generateUuid(), + finishReason: jsonData.finish_reason, + }; +} + +// Helper functions +function calculateMeanLogProb(accessor: ServicesAccessor, jsonData: APIJsonData): number | undefined { + if (!jsonData?.logprobs?.token_logprobs) { + return undefined; + } + + try { + let logProbSum = 0.0; + let numTokens = 0; + + // Limit to first 50 logprobs, avoids up-ranking longer solutions + let iterLimit = 50; + + // First token is always null and last token can have multiple options if it hit a stop + for (let i = 0; i < jsonData.logprobs.token_logprobs.length - 1 && iterLimit > 0; i++, iterLimit--) { + logProbSum += jsonData.logprobs.token_logprobs[i]; + numTokens += 1; + } + + if (numTokens > 0) { + return logProbSum / numTokens; + } else { + return undefined; + } + } catch (e) { + logger.exception(accessor, e, `Error calculating mean prob`); + } +} + +function calculateMeanAlternativeLogProb(accessor: ServicesAccessor, jsonData: APIJsonData): number | undefined { + if (!jsonData?.logprobs?.top_logprobs) { + return undefined; + } + + try { + let logProbSum = 0.0; + let numTokens = 0; + + // Limit to first 50 logprobs, avoids up-ranking longer solutions + let iterLimit = 50; + + for (let i = 0; i < jsonData.logprobs.token_logprobs.length - 1 && iterLimit > 0; i++, iterLimit--) { + // copy the options object to avoid mutating the original + const options = { ...jsonData.logprobs.top_logprobs[i] }; + delete options[jsonData.logprobs.tokens[i]]; + logProbSum += Math.max(...Object.values(options)); + numTokens += 1; + } + + if (numTokens > 0) { + return logProbSum / numTokens; + } else { + return undefined; + } + } catch (e) { + logger.exception(accessor, e, `Error calculating mean prob`); + } +} + +// Returns a temperature in range 0.0-1.0, using either a config setting, +// or the following ranges: 1=0.0, <10=0.2, <20=0.4, >=20=0.8 +export function getTemperatureForSamples(runtime: ICompletionsRuntimeModeService, numShots: number): number { + if (runtime.isRunningInTest()) { + return 0.0; + } + + if (numShots <= 1) { + return 0.0; + } else if (numShots < 10) { + return 0.2; + } else if (numShots < 20) { + return 0.4; + } else { + return 0.8; + } +} + +const stopsForLanguage: { [key: string]: string[] } = { + markdown: ['\n\n\n'], + python: ['\ndef ', '\nclass ', '\nif ', '\n\n#'], +}; + +export function getStops(languageId?: string) { + return stopsForLanguage[languageId ?? ''] ?? ['\n\n\n', '\n```']; +} + +export function getTopP(): number { + return 1; +} + +export function getMaxSolutionTokens(): number { + return DEFAULT_MAX_COMPLETION_LENGTH; +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/stream.ts b/completions-sample-code/vscode-node/lib/src/openai/stream.ts new file mode 100644 index 0000000..ece1bbb --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/stream.ts @@ -0,0 +1,722 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotAnnotation, CopilotAnnotations, CopilotNamedAnnotationList, StreamCopilotAnnotations } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { getRequestId, RequestId } from '../../../../../../platform/networking/common/fetch'; +import { DestroyableStream } from '../../../../../../platform/networking/common/fetcherService'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CancellationToken as ICancellationToken } from '../../../types/src'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { Response } from '../networking'; +import { TelemetryWithExp } from '../telemetry'; +import { getEngineRequestInfo } from './config'; +import { CopilotConfirmation, CopilotError, CopilotReference, SolutionDecision } from './fetch'; +import { + APIChoice, + APIJsonData, + APILogprobs, + convertToAPIChoice, + FinishedCallback, +} from './openai'; + +const streamChoicesLogger = new Logger('streamChoices'); + +/** Gathers together many chunks of a single completion choice. */ +class APIJsonDataStreaming { + logprobs: number[][] = []; + top_logprobs: { [key: string]: number }[][] = []; + text: string[] = []; + tokens: string[][] = []; + text_offset: number[][] = []; + copilot_annotations: CopilotAnnotations = new StreamCopilotAnnotations(); + tool_calls: StreamingToolCalls = new StreamingToolCalls(); + function_call: StreamingFunctionCall = new StreamingFunctionCall(); + copilot_references: CopilotReference[] = []; + finish_reason?: string; + yielded = false; + + append(choice: ChoiceJSON) { + if (choice.text) { + this.text.push(choice.text); + } + // Role function is not included in the main answer. + if (choice.delta?.content && choice.delta.role !== 'function') { + this.text.push(choice.delta.content); + } + if (choice.logprobs) { + this.tokens.push(choice.logprobs.tokens ?? []); + this.text_offset.push(choice.logprobs.text_offset ?? []); + this.logprobs.push(choice.logprobs.token_logprobs ?? []); + this.top_logprobs.push(choice.logprobs.top_logprobs ?? []); + } + if (choice.copilot_annotations) { + this.copilot_annotations.update(choice.copilot_annotations); + } + if (choice.delta?.copilot_annotations) { + this.copilot_annotations.update(choice.delta.copilot_annotations); + } + if (choice.delta?.tool_calls && choice.delta.tool_calls.length > 0) { + this.tool_calls.update(choice.delta.tool_calls); + } + if (choice.delta?.function_call) { + this.function_call.update(choice.delta.function_call); + } + if (choice?.finish_reason) { + this.finish_reason = choice.finish_reason; + } + } +} + +// Given a string of lines separated by one or more newlines, returns complete +// lines and any remaining partial line data. Exported for test only. +export function splitChunk(chunk: string): [string[], string] { + const dataLines = chunk.split('\n'); + const newExtra = dataLines.pop(); // will be empty string if chunk ends with "\n" + return [dataLines.filter(line => line !== ''), newExtra!]; +} + +type ModelUsage = { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; +}; + +/** + * A single finished completion returned from the model or proxy, along with + * some metadata. + */ +export interface FinishedCompletion { + solution: APIJsonDataStreaming; + /** An optional offset into `solution.text.join('')` where the completion finishes. */ + finishOffset: number | undefined; + /** A copilot-specific human-readable reason for the completion finishing. */ + reason: string | null; + requestId: RequestId; + index: number; + model?: string; + usage?: ModelUsage; +} + +class StreamingToolCall { + // Right now we only support functions. + name?: string; + arguments: string[] = []; + id?: string; // Unique ID for the tool call, if available + + update(toolCall: { type: 'function'; id?: string; function: { name?: string; arguments: string } }) { + if (toolCall.id) { + this.id = toolCall.id; + } + if (toolCall.function.name) { + this.name = toolCall.function.name; + } + this.arguments.push(toolCall.function.arguments); + } +} + +class StreamingToolCalls { + private toolCalls: StreamingToolCall[] = []; + + constructor() { } + + update( + toolCallsArray: { type: 'function'; id?: string; index?: number; function: { name?: string; arguments: string } }[] + ) { + toolCallsArray.forEach(toolCall => { + let currentCall = this.toolCalls.length > 0 ? this.toolCalls[this.toolCalls.length - 1] : undefined; + // Create a new tool call if: + // 1. No existing tool calls, OR + // 2. The new tool call has an ID and it's different from the current one + if (!currentCall || (toolCall.id && currentCall.id !== toolCall.id)) { + currentCall = new StreamingToolCall(); + this.toolCalls.push(currentCall); + } + + currentCall.update(toolCall); + }); + } + + getToolCalls(): StreamingToolCall[] { + return this.toolCalls; + } +} + +class StreamingFunctionCall { + name?: string; + arguments: string[] = []; + + update(functionCall: { name?: string; arguments: string }) { + if (functionCall.name) { + this.name = functionCall.name; + } + this.arguments.push(functionCall.arguments); + } +} + +interface FunctionCallJSON { + name?: string; + arguments: string; +} + +interface ToolCallJSON { + id: string; + function: FunctionCallJSON; + index: number; + type: 'function'; +} + +/** What comes back from the OpenAI API for a single choice in an SSE chunk. */ +interface ChoiceJSON { + index: number; + /** + * The text attribute as defined in completions streaming. + * See https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format + */ + text: string; + copilot_annotations: { [key: string]: CopilotAnnotation[] }; + /** + * The delta attribute as defined in chat streaming. + * See https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb + */ + delta: { + content: string; + copilot_annotations?: { [key: string]: CopilotAnnotation[] }; + role?: string; + function_call?: FunctionCallJSON; + tool_calls?: ToolCallJSON[]; + }; + finish_reason: string | null; + logprobs?: APILogprobs; + copilot_annotation?: CopilotNamedAnnotationList; + copilot_references?: CopilotReference[]; +} + +/** + * Processes an HTTP request containing what is assumed to be an SSE stream of + * OpenAI API data. Yields a stream of `FinishedCompletion` objects, each as + * soon as it's finished. + */ +export class SSEProcessor { + private requestId: RequestId = getRequestId(this.response.headers); + private stats = new ChunkStats(); + /** + * A key & value being here means at least one chunk with that choice index + * has been received. A null value means we've already finished the given + * solution and should not process incoming tokens further. + */ + private readonly solutions: Record<number, APIJsonDataStreaming | null> = {}; + + private constructor( + private readonly expectedNumChoices: number, + private readonly response: Response, + private readonly body: DestroyableStream<string>, + private readonly telemetryData: TelemetryWithExp, + private readonly dropCompletionReasons: string[], + private readonly cancellationToken: ICancellationToken | undefined = undefined, + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsLogTargetService private readonly logTarget: ICompletionsLogTargetService, + ) { } + + /** + * Creates a new instance of SSEProcessor. + * + * Supports dropping completions with specific finish reasons. + * Historically, this was used to drop RAI ('content_filter') completions, instead of showing partially finished completions to the user. We've gone back and forth on this. + */ + static async create( + accessor: ServicesAccessor, + expectedNumChoices: number, + response: Response, + telemetryData: TelemetryWithExp, + dropCompletionReasons?: string[], + cancellationToken?: ICancellationToken + ) { + const instantiationService = accessor.get(IInstantiationService); + const logTargetService = accessor.get(ICompletionsLogTargetService); + + const body = response.body.pipeThrough(new TextDecoderStream()); + + // TODO@benibenj can we switch to our SSEProcessor implementation? + // It seems like they build more on top of the shared impl + // I made this function async and commented out the web ReadableStream approach + + return new SSEProcessor( + expectedNumChoices, + response, + body, + telemetryData, + dropCompletionReasons ?? [], + cancellationToken, + instantiationService, + logTargetService, + ); + } + + /** + * Yields finished completions as soon as they are available. The finishedCb + * is used to determine when a completion is done and should be truncated. + * It is called on the whole of the received solution text, once at the end + * of the completion (if it stops by itself) and also on any chunk that has + * a newline in it. + * + * Closes the server request stream when all choices are finished/truncated. + * + * Note that for this to work, the caller must consume the entire stream. + * This happens automatically when using a `for await` loop, but when + * iterating manually this needs to be done by calling `.next()` until it + * returns an item with done = true (or calling `.return()`). + */ + async *processSSE(finishedCb: FinishedCallback = () => undefined): AsyncIterable<FinishedCompletion> { + try { + yield* this.processSSEInner(finishedCb); + } finally { + await this.cancel(); + streamChoicesLogger.debug(this.logTarget, + `request done: headerRequestId: [${this.requestId.headerRequestId}] model deployment ID: [${this.requestId.deploymentId}]` + ); + streamChoicesLogger.debug(this.logTarget, 'request stats:', this.stats); + } + } + + private async *processSSEInner(finishedCb: FinishedCallback): AsyncIterable<FinishedCompletion> { + // Collects pieces of the SSE stream that haven't been fully processed + // yet. + let extraData = ''; + + let currentFinishReason: string | null = null; + let model: string | undefined; + let usage: ModelUsage | undefined; + + // Iterate over arbitrarily sized chunks coming in from the network. + networkRead: for await (const chunk of this.body) { + if (await this.maybeCancel('after awaiting body chunk')) { + return; + } + + streamChoicesLogger.debug(this.logTarget, 'chunk', chunk.toString()); + const [dataLines, remainder] = splitChunk(extraData + chunk.toString()); + extraData = remainder; + + // Each dataLine is complete since we've seen at least one \n after + // it. + for (const dataLine of dataLines) { + const lineWithoutData = dataLine.slice('data:'.length).trim(); + if (lineWithoutData === '[DONE]') { + yield* this.finishSolutions(currentFinishReason, model, usage, finishedCb); + return; + } + // If this is not a DONE line, we reset the finish reason. + currentFinishReason = null; + + interface StreamingResponse { + choices?: ChoiceJSON[]; + error?: { message: string }; + copilot_references?: CopilotReference[]; + copilot_confirmation?: unknown; + copilot_errors: CopilotError[]; + model?: string; // Note: model should only be expected from CAPI, not copilot-proxy + usage?: ModelUsage; + } + + let json; + try { + json = <StreamingResponse>JSON.parse(lineWithoutData); + } catch (e) { + streamChoicesLogger.error(this.logTarget, 'Error parsing JSON stream data', dataLine); + continue; + } + + // A message with a confirmation may or may not have 'choices' + if (json.copilot_confirmation && isCopilotConfirmation(json.copilot_confirmation)) { + await finishedCb('', { + text: '', + requestId: this.requestId, + copilotConfirmation: json.copilot_confirmation, + }); + } + + // we do not process the data from role=function right now because copilot_references seem to contain the same data in a more structured way + if (json.copilot_references) { + await finishedCb('', { + text: '', + requestId: this.requestId, + copilotReferences: json.copilot_references, + }); + } + + if (json.choices === undefined) { + if (!json.copilot_references && !json.copilot_confirmation) { + if (json.error !== undefined) { + streamChoicesLogger.error(this.logTarget, 'Error in response:', json.error!.message); + } else { + streamChoicesLogger.error(this.logTarget, + 'Unexpected response with no choices or error: ' + lineWithoutData + ); + } + } + + // There are also messages with a null 'choices' that include copilot_errors- report these + if (json.copilot_errors) { + await finishedCb('', { text: '', requestId: this.requestId, copilotErrors: json.copilot_errors }); + } + + continue; + } + + if (model === undefined && json.model) { + model = json.model; + } + + if (usage === undefined && json.usage) { + usage = json.usage; + } + + if (this.allSolutionsDone()) { + // discard any extra data; there's no need to log it as an error + extraData = ''; + break networkRead; + } + + for (let i = 0; i < json.choices?.length; i++) { + const choice: ChoiceJSON = json.choices[i]; + streamChoicesLogger.debug(this.logTarget, 'choice', choice); + this.stats.add(choice.index); + + if (!(choice.index in this.solutions)) { + this.solutions[choice.index] = new APIJsonDataStreaming(); + } + + const solution = this.solutions[choice.index]; + if (solution === null) { + continue; // already finished + } + + solution.append(choice); + + // Call finishedCb after each newline token to determine + // if the solution is now complete. Also call it if the + // solution has finished to make sure it's properly truncated. + let decision = this.asSolutionDecision(); + const hasNewLine = choice.text?.indexOf('\n') > -1 || choice.delta?.content?.indexOf('\n') > -1; + if (choice.finish_reason || hasNewLine) { + const text = solution.text.join(''); + decision = this.asSolutionDecision( + await finishedCb(text, { + text, + index: choice.index, + requestId: this.requestId, + annotations: solution.copilot_annotations, + copilotReferences: solution.copilot_references, + getAPIJsonData: () => convertToAPIJsonData(solution), + finished: choice.finish_reason ? true : false, + telemetryData: this.telemetryData, + }) + ); + + if (await this.maybeCancel('after awaiting finishedCb')) { + return; + } + } + + /** + * If this is a function call and we have a finish reason, continue to the next choice. + * This is because of how extensibility platform agents work, where multiple finish reasons can be returned. + * + * This should be updated to tools in the future. + */ + if (choice.finish_reason && solution.function_call.name !== undefined) { + currentFinishReason = choice.finish_reason; + continue; + } + + if (choice.finish_reason) { + decision.yieldSolution = true; + decision.continueStreaming = false; + } + if (!decision.yieldSolution) { + continue; + } + // NOTE: When there is a finish_reason the text of subsequent chunks is always '', + // (current chunk might still have useful text, that is why we add it above). + // So we know that we already got all the text to be displayed for the user. + // TODO: This might contain additional logprobs for excluded next tokens. We should + // filter out indices that correspond to excluded tokens. It will not affect the + // text though. + const loggedReason = choice.finish_reason ?? 'client-trimmed'; + streamChoicesLogger.debug(this.logTarget, + 'completion.finishReason', + this.telemetryData.extendedBy({ + completionChoiceFinishReason: loggedReason, + engineName: model ?? '', + engineChoiceSource: this.instantiationService.invokeFunction(getEngineRequestInfo, this.telemetryData).engineChoiceSource, + }) + ); + if (this.dropCompletionReasons.includes(choice.finish_reason!)) { + // In this case we drop the choice on the floor. + this.solutions[choice.index] = null; + } else if (!solution.yielded) { + this.stats.markYielded(choice.index); + yield { + solution, + finishOffset: decision.finishOffset, + reason: choice.finish_reason, + requestId: this.requestId, + index: choice.index, + model: model, + usage: usage, + }; + solution.yielded = true; + } + + if (await this.maybeCancel('after yielding finished choice')) { + return; + } + + if (!decision.continueStreaming) { + this.solutions[choice.index] = null; + } + } + } + } + + // Yield whatever solutions remain incomplete in case no [DONE] was received. + // This shouldn't happen in practice unless there was an error somewhere. + for (const [index, solution] of Object.entries(this.solutions)) { + const solutionIndex = Number(index); // Convert `index` from string to number + if (solution === null) { + continue; // already finished + } + streamChoicesLogger.debug(this.logTarget, + 'completion.finishReason', + this.telemetryData.extendedBy({ + completionChoiceFinishReason: 'Iteration Done', + engineName: model ?? '', + }) + ); + this.stats.markYielded(solutionIndex); + yield { + solution, + finishOffset: undefined, + reason: 'Iteration Done', + requestId: this.requestId, + index: solutionIndex, + model: model, + usage: usage, + }; + + if (await this.maybeCancel('after yielding after iteration done')) { + return; + } + } + + // Error message can be present in `extraData` + if (extraData.length > 0) { + try { + const extraDataJson = <{ error?: { message: string } }>JSON.parse(extraData); + if (extraDataJson.error !== undefined) { + streamChoicesLogger.error(this.logTarget, + `Error in response: ${extraDataJson.error!.message}`, + extraDataJson.error + ); + } + } catch (e) { + streamChoicesLogger.error(this.logTarget, `Error parsing extraData: ${extraData}`); + } + } + } + + private asSolutionDecision(result?: SolutionDecision | number): SolutionDecision { + if (result === undefined) { + return { + yieldSolution: false, + continueStreaming: true, + }; + } else if (typeof result === 'number') { + return { + yieldSolution: true, + continueStreaming: false, + finishOffset: result, + }; + } + + return result; + } + + /** Yields the solutions that weren't yet finished, with a 'DONE' reason. */ + private async *finishSolutions( + currentFinishReason: string | null, + model: string | undefined, + usage: ModelUsage | undefined, + finishedCb: FinishedCallback + ): AsyncIterable<FinishedCompletion> { + for (const [index, solution] of Object.entries(this.solutions)) { + const solutionIndex = Number(index); // Convert `index` from string to number + if (solution === null) { + continue; // already finished + } + // ensure the callback receives the final result + const text = solution.text.join(''); + await finishedCb(text, { + text, + index: solutionIndex, + requestId: this.requestId, + annotations: solution.copilot_annotations, + copilotReferences: solution.copilot_references, + getAPIJsonData: () => convertToAPIJsonData(solution), // observation from @ulugbekna: this conversion will make `finishReason` for this object 'stop' while we're yielding with 'DONE' below + finished: true, + telemetryData: this.telemetryData, + }); + if (solution.yielded) { + continue; // already produced + } + this.stats.markYielded(solutionIndex); + streamChoicesLogger.debug(this.logTarget, + 'completion.finishReason', + this.telemetryData.extendedBy({ + completionChoiceFinishReason: currentFinishReason ?? 'DONE', + engineName: model ?? '', + }) + ); + yield { + solution, + finishOffset: undefined, + reason: currentFinishReason ?? 'DONE', + requestId: this.requestId, + index: solutionIndex, + model: model, + usage: usage, + }; + + if (await this.maybeCancel('after yielding on DONE')) { + return; + } + } + } + + /** + * Returns whether the cancellation token was cancelled and closes the + * stream if it was. + */ + private async maybeCancel(description: string) { + if (this.cancellationToken?.isCancellationRequested) { + streamChoicesLogger.debug(this.logTarget, 'Cancelled: ' + description); + await this.cancel(); + return true; + } + return false; + } + + /** Cancels the network request to the proxy. */ + private async cancel() { + await this.body.destroy(); + } + + /** Returns whether we've finished receiving all expected solutions. */ + private allSolutionsDone(): boolean { + const solutions = Object.values(this.solutions); + return solutions.length === this.expectedNumChoices && solutions.every(s => s === null); + } +} + +export function prepareSolutionForReturn( + accessor: ServicesAccessor, + c: FinishedCompletion, + telemetryData: TelemetryWithExp +): APIChoice { + const logTarget = accessor.get(ICompletionsLogTargetService); + let completionText = c.solution.text.join(''); + + let blockFinished = false; + if (c.finishOffset !== undefined) { + // Trim solution to finishOffset returned by finishedCb + streamChoicesLogger.debug(logTarget, `solution ${c.index}: early finish at offset ${c.finishOffset}`); + completionText = completionText.substring(0, c.finishOffset); + blockFinished = true; + } + + streamChoicesLogger.info(logTarget, `solution ${c.index} returned. finish reason: [${c.reason}]`); + streamChoicesLogger.debug(logTarget, `solution ${c.index} details: finishOffset: [${c.finishOffset}]`); + const jsonData: APIJsonData = convertToAPIJsonData(c.solution); + return convertToAPIChoice(accessor, completionText, jsonData, c.index, c.requestId, blockFinished, telemetryData); +} + +// Function to convert from APIJsonDataStreaming to APIJsonData format +function convertToAPIJsonData(streamingData: APIJsonDataStreaming): APIJsonData { + const joinedText = streamingData.text.join(''); + const annotations = streamingData.copilot_annotations.current; + const out: APIJsonData = { + text: joinedText, + tokens: streamingData.text, + copilot_annotations: annotations, + finish_reason: streamingData.finish_reason ?? 'stop', + }; + if (streamingData.logprobs.length === 0) { + return out; + } + const flattenedLogprobs = streamingData.logprobs.reduce((acc, cur) => acc.concat(cur), []); + const flattenedTopLogprobs = streamingData.top_logprobs.reduce((acc, cur) => acc.concat(cur), []); + const flattenedOffsets = streamingData.text_offset.reduce((acc, cur) => acc.concat(cur), []); + const flattenedTokens = streamingData.tokens.reduce((acc, cur) => acc.concat(cur), []); + + return { + ...out, + logprobs: { + token_logprobs: flattenedLogprobs, + top_logprobs: flattenedTopLogprobs, + text_offset: flattenedOffsets, + tokens: flattenedTokens, + }, + }; +} + +// data: {"choices":null,"copilot_confirmation":{"type":"action","title":"Are you sure you want to proceed?","message":"This action is irreversible.","confirmation":{"id":"123"}},"id":null} +function isCopilotConfirmation(obj: unknown): obj is CopilotConfirmation { + return ( + typeof (obj as CopilotConfirmation).title === 'string' && + typeof (obj as CopilotConfirmation).message === 'string' && + !!(obj as CopilotConfirmation).confirmation + ); +} + +/** Keeps track of how many chunks of a choice were read and yielded out. */ +class ChunkStats { + private readonly choices = new Map<number, ChoiceStats>(); + + private getChoiceStats(choiceIndex: number): ChoiceStats { + let choiceStat = this.choices.get(choiceIndex); + if (!choiceStat) { + choiceStat = new ChoiceStats(); + this.choices.set(choiceIndex, choiceStat); + } + return choiceStat; + } + + add(choiceIndex: number) { + this.getChoiceStats(choiceIndex).increment(); + } + + markYielded(choiceIndex: number) { + this.getChoiceStats(choiceIndex).markYielded(); + } + + toString() { + return Array.from(this.choices.entries()) + .map(([index, stats]) => `${index}: ${stats.yieldedTokens} -> ${stats.seenTokens}`) + .join(', '); + } +} + +class ChoiceStats { + yieldedTokens = -1; + seenTokens = 0; + + increment() { + this.seenTokens++; + } + + markYielded() { + this.yieldedTokens = this.seenTokens; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/test/config.test.ts b/completions-sample-code/vscode-node/lib/src/openai/test/config.test.ts new file mode 100644 index 0000000..fe62a3d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/test/config.test.ts @@ -0,0 +1,29 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ExpTreatmentVariables } from '../../experiments/expConfig'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { getEngineRequestInfo } from '../config'; + +suite('OpenAI Config Tests', function () { + let accessor: ServicesAccessor; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + }); + + test('getEngineRequestInfo() returns the model from AvailableModelManager', function () { + const telem = TelemetryWithExp.createEmptyConfigForTesting(); + telem.filtersAndExp.exp.variables[ExpTreatmentVariables.CustomEngine] = 'model.override'; + + const info = getEngineRequestInfo(accessor, telem); + + assert.strictEqual(info.modelId, 'model.override'); + assert.deepStrictEqual(info.headers, {}); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/openai/test/fetch.test.ts b/completions-sample-code/vscode-node/lib/src/openai/test/fetch.test.ts new file mode 100644 index 0000000..9c3fbf1 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/test/fetch.test.ts @@ -0,0 +1,402 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as Sinon from 'sinon'; +import { TestingServiceCollection } from '../../../../../../../platform/test/node/services'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; +import { SyncDescriptor } from '../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CancellationTokenSource } from '../../../../types/src'; +import { ICompletionsCopilotTokenManager } from '../../auth/copilotTokenManager'; +import { FetchOptions, ICompletionsFetcherService, Response } from '../../networking'; +import { ICompletionsStatusReporter, StatusChangedEvent, StatusReporter } from '../../progress'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { createFakeResponse, createFakeStreamResponse, StaticFetcher } from '../../test/fetcher'; +import { withInMemoryTelemetry } from '../../test/telemetry'; +import { + CMDQuotaExceeded, + CompletionParams, + CopilotUiKind, + ICompletionsOpenAIFetcherService, + LiveOpenAIFetcher, sanitizeRequestOptionTelemetry +} from '../fetch'; +import { ErrorReturningFetcher, SyntheticCompletions } from '../fetch.fake'; + +suite('"Fetch" unit tests', function () { + let accessor: ServicesAccessor; + let serviceCollection: TestingServiceCollection; + let resetSpy: Sinon.SinonSpy<Parameters<ICompletionsCopilotTokenManager['resetToken']>>; + + setup(function () { + serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsOpenAIFetcherService, new SyncDescriptor(ErrorReturningFetcher)); + accessor = serviceCollection.createTestingAccessor(); + resetSpy = Sinon.spy(accessor.get(ICompletionsCopilotTokenManager), 'resetToken'); + }); + + test('Empty/whitespace completions are stripped', async function () { + const fetcher = new SyntheticCompletions(['', ' ', '\n'], accessor.get(ICompletionsCopilotTokenManager)); + const params: CompletionParams = { + prompt: { + prefix: '', + suffix: '', + isFimEnabled: false, + }, + languageId: '', + repoInfo: undefined, + engineModelId: '', + count: 1, + uiKind: CopilotUiKind.GhostText, + ourRequestId: generateUuid(), + extra: {}, + }; + const cancellationToken = new CancellationTokenSource().token; + const res = await fetcher.fetchAndStreamCompletions( + params, + TelemetryWithExp.createEmptyConfigForTesting(), + () => undefined, + cancellationToken + ); + assert.deepStrictEqual(res.type, 'success'); + // keep the type checker happy + if (res.type !== 'success') { + throw new Error(`internal error: res.type is not 'success'`); + } + const stream = res.choices; + const results = []; + for await (const result of stream) { + results.push(result); + } + assert.strictEqual(results.length, 0); + }); + + test('If in the split context experiment, send the context field as part of the request', async function () { + const networkFetcher = new OptionsRecorderFetcher(() => createFakeStreamResponse('data: [DONE]\n')); + const params: CompletionParams = { + prompt: { + context: ['# Language: Python'], + prefix: 'prefix without context', + suffix: '\ndef sum(a, b):\n return a + b', + isFimEnabled: true, + }, + languageId: 'python', + repoInfo: undefined, + engineModelId: 'copilot-codex', + count: 1, + uiKind: CopilotUiKind.GhostText, + postOptions: {}, + ourRequestId: generateUuid(), + extra: {}, + }; + + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define(ICompletionsFetcherService, networkFetcher); + const accessor = serviceCollectionClone.createTestingAccessor(); + + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables.copilotenablepromptcontextproxyfield = true; + + const openAIFetcher = accessor.get(IInstantiationService).createInstance(LiveOpenAIFetcher); + await openAIFetcher.fetchAndStreamCompletions(params, telemetryWithExp, () => undefined); + + const options = networkFetcher.options; + const json = options?.json as Record<string, unknown> | undefined; + assert.strictEqual(json?.prompt, params.prompt.prefix); + const extra = json?.extra as Record<string, unknown> | undefined; + assert.strictEqual(extra?.context, params.prompt.context); + }); + + test('properly handles 466 (client outdated) responses from proxy', async function () { + const statusReporter = new TestStatusReporter(); + const result = await assertResponseWithStatus(466, statusReporter); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'client not supported: response-text' }); + assert.deepStrictEqual(statusReporter.kind, 'Error'); + assert.deepStrictEqual(statusReporter.message, 'response-text'); + assert.deepStrictEqual(statusReporter.eventCount, 1); + }); + + test('has fallback for unknown http response codes from proxy', async function () { + const statusReporter = new TestStatusReporter(); + const result = await assertResponseWithStatus(518, statusReporter); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'unhandled status from server: 518 response-text' }); + assert.deepStrictEqual(statusReporter.kind, 'Warning'); + assert.deepStrictEqual(statusReporter.message, 'Last response was a 518 error'); + }); + + test('calls out possible proxy for 4xx requests without x-github-request-id', async function () { + const statusReporter = new TestStatusReporter(); + const result = await assertResponseWithStatus(418, statusReporter, { 'x-github-request-id': '' }); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'unhandled status from server: 418 response-text' }); + assert.deepStrictEqual(statusReporter.kind, 'Warning'); + assert.deepStrictEqual( + statusReporter.message, + 'Last response was a 418 error and does not appear to originate from GitHub. Is a proxy or firewall intercepting this request? https://gh.io/copilot-firewall' + ); + }); + + test('HTTP `Unauthorized` invalidates token', async function () { + const result = await assertResponseWithContext(accessor, 401); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'token expired or invalid: 401' }); + assert.ok(resetSpy.calledOnce, 'resetToken should have been called once'); + }); + + test('HTTP `Forbidden` invalidates token', async function () { + const result = await assertResponseWithContext(accessor, 403); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'token expired or invalid: 403' }); + assert.ok(resetSpy.calledOnce, 'resetToken should have been called once'); + }); + + test('HTTP `Too many requests` enforces rate limiting locally', async function () { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsOpenAIFetcherService, new SyncDescriptor(ErrorReturningFetcher)); + const accessor = serviceCollection.createTestingAccessor(); + const result = await assertResponseWithContext(accessor, 429); + const fetcherService = accessor.get(ICompletionsOpenAIFetcherService); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'rate limited' }); + const limited = await fetcherService.fetchAndStreamCompletions( + {} as CompletionParams, + TelemetryWithExp.createEmptyConfigForTesting(), + () => Promise.reject(new Error()), + new CancellationTokenSource().token + ); + assert.deepStrictEqual(limited, { type: 'canceled', reason: 'rate limited' }); + }); + + test.skip('properly handles 402 (free plan exhausted) responses from proxy', async function () { + const fetcherService = accessor.get(ICompletionsOpenAIFetcherService); + const tokenManager = accessor.get(ICompletionsCopilotTokenManager); + await tokenManager.primeToken(); // Trigger initial status + const statusReporter = new TestStatusReporter(); + + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define(ICompletionsStatusReporter, statusReporter); + const accessorClone = serviceCollectionClone.createTestingAccessor(); + const result = await assertResponseWithContext(accessorClone, 402); + + assert.deepStrictEqual(result, { type: 'failed', reason: 'monthly free code completions exhausted' }); + assert.deepStrictEqual(statusReporter.kind, 'Error'); + assert.match(statusReporter.message, /limit/); + assert.deepStrictEqual(statusReporter.eventCount, 1); + assert.deepStrictEqual(statusReporter.command, CMDQuotaExceeded); + const exhausted = await fetcherService.fetchAndStreamCompletions( + fakeCompletionParams(), + TelemetryWithExp.createEmptyConfigForTesting(), + () => Promise.reject(new Error()), + new CancellationTokenSource().token + ); + assert.deepStrictEqual(exhausted, { type: 'canceled', reason: 'monthly free code completions exhausted' }); + + tokenManager.resetToken(); + await tokenManager.getToken(); + + const refreshed = await assertResponseWithContext(accessorClone, 429); + assert.deepStrictEqual(refreshed, { type: 'failed', reason: 'rate limited' }); + assert.deepStrictEqual(statusReporter.kind, 'Error'); + }); + + test('additional headers are included in the request', async function () { + const networkFetcher = new StaticFetcher(() => createFakeStreamResponse('data: [DONE]\n')); + const params: CompletionParams = { + prompt: { + prefix: '', + suffix: '', + isFimEnabled: false, + }, + languageId: '', + repoInfo: undefined, + engineModelId: 'copilot-codex', + count: 1, + uiKind: CopilotUiKind.GhostText, + ourRequestId: generateUuid(), + headers: { Host: 'bla' }, + extra: {}, + }; + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define(ICompletionsFetcherService, networkFetcher); + const accessor = serviceCollectionClone.createTestingAccessor(); + + const openAIFetcher = accessor.get(IInstantiationService).createInstance(LiveOpenAIFetcher); + await openAIFetcher.fetchAndStreamCompletions( + params, + TelemetryWithExp.createEmptyConfigForTesting(), + () => undefined + ); + + assert.strictEqual(networkFetcher.headerBuffer!['Host'], 'bla'); + }); + +}); + +suite('Telemetry sent on fetch', function () { + let accessor: ServicesAccessor; + + setup(function () { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsFetcherService, new OptionsRecorderFetcher(() => createFakeStreamResponse('data: [DONE]\n'))); + accessor = serviceCollection.createTestingAccessor(); + }); + + test('sanitizeRequestOptionTelemetry properly excludes top-level keys', function () { + const request = { + prompt: 'prompt prefix', + suffix: 'prompt suffix', + stream: true as const, + count: 1, + extra: { + language: 'python', + }, + }; + + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + + sanitizeRequestOptionTelemetry(request, telemetryWithExp, ['prompt', 'suffix']); + + assert.deepStrictEqual(telemetryWithExp.properties, { + 'request.option.stream': 'true', + 'request.option.count': '1', + 'request.option.extra': '{"language":"python"}', + }); + }); + + test('sanitizeRequestOptionTelemetry properly excludes `extra` keys', function () { + const request = { + prompt: 'prefix without context', + suffix: 'prompt suffix', + stream: true as const, + count: 1, + extra: { + language: 'python', + context: ['# Language: Python'], + }, + }; + + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + + sanitizeRequestOptionTelemetry(request, telemetryWithExp, ['prompt', 'suffix'], ['context']); + + assert.deepStrictEqual(telemetryWithExp.properties, { + 'request.option.stream': 'true', + 'request.option.count': '1', + 'request.option.extra': '{"language":"python"}', + }); + }); + + test('If context is provided while in the split context experiment, only send it in restricted telemetry events', async function () { + const params: CompletionParams = { + prompt: { + context: ['# Language: Python'], + prefix: 'prefix without context', + suffix: '\ndef sum(a, b):\n return a + b', + isFimEnabled: true, + }, + languageId: 'python', + repoInfo: undefined, + engineModelId: 'copilot-codex', + count: 1, + uiKind: CopilotUiKind.GhostText, + postOptions: {}, + ourRequestId: generateUuid(), + extra: {}, + }; + + const openAIFetcher = accessor.get(IInstantiationService).createInstance(LiveOpenAIFetcher); + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables.copilotenablepromptcontextproxyfield = true; + + const { reporter } = await withInMemoryTelemetry(accessor, async () => { + await openAIFetcher.fetchAndStreamCompletions(params, telemetryWithExp, () => undefined); + }); + + const standardEvents = reporter.events; + const hasContext = standardEvents.some(event => event.properties['request_option_extra']?.includes('context')); + assert.strictEqual(hasContext, false, 'Standard telemetry event should not include context'); + + // todo@dbaeumer we need to understand what our restricted telemetry story is. + // const restrictedEvents = enhancedReporter.events; + // const hasRestrictedContext = restrictedEvents.some(event => + // event.properties['request_option_extra']?.includes('context') + // ); + // assert.strictEqual(hasRestrictedContext, true, 'Restricted telemetry event should include context'); + }); + + test('If context is provided, include it in `engine.prompt` telemetry events', function () { }); +}); + +class TestStatusReporter extends StatusReporter { + eventCount = 0; + kind = 'Normal'; + message = ''; + command: string | undefined; + + override didChange(event: StatusChangedEvent): void { + this.eventCount++; + this.kind = event.kind; + this.message = event.message || ''; + this.command = event.command?.command; + } +} + +async function assertResponseWithStatus( + statusCode: number, + statusReporter: ICompletionsStatusReporter, + headers?: Record<string, string> +) { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsStatusReporter, statusReporter); + const accessor = serviceCollection.createTestingAccessor(); + const copilotTokenManager = accessor.get(ICompletionsCopilotTokenManager); + await copilotTokenManager.primeToken(); // Trigger initial status + return assertResponseWithContext(accessor, statusCode, headers); +} + +async function assertResponseWithContext(accessor: ServicesAccessor, statusCode: number, headers?: Record<string, string>) { + const response = createFakeResponse(statusCode, 'response-text', headers); + const fetcher = accessor.getIfExists(ICompletionsOpenAIFetcherService) as ErrorReturningFetcher ?? accessor.get(IInstantiationService).createInstance(ErrorReturningFetcher); + fetcher.setResponse(response); + const completionParams: CompletionParams = fakeCompletionParams(); + const result = await fetcher.fetchAndStreamCompletions( + completionParams, + TelemetryWithExp.createEmptyConfigForTesting(), + () => Promise.reject(new Error()), + new CancellationTokenSource().token + ); + return result; +} + +function fakeCompletionParams(): CompletionParams { + return { + prompt: { + prefix: 'xxx', + suffix: '', + isFimEnabled: false, + }, + languageId: '', + repoInfo: undefined, + ourRequestId: generateUuid(), + engineModelId: 'foo/bar', + count: 1, + uiKind: CopilotUiKind.GhostText, + postOptions: {}, + extra: {}, + }; +} + +class OptionsRecorderFetcher extends StaticFetcher { + options: FetchOptions | undefined; + + override fetch(url: string, options: FetchOptions): Promise<Response> { + this.options = options; + + return super.fetch(url, options); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/openai/test/stream.test.ts b/completions-sample-code/vscode-node/lib/src/openai/test/stream.test.ts new file mode 100644 index 0000000..81887ec --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/openai/test/stream.test.ts @@ -0,0 +1,821 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { CopilotAnnotation, StreamCopilotAnnotations } from '../../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { asyncIterableToArray } from '../../helpers/iterableHelpers'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { createFakeStreamResponse } from '../../test/fetcher'; +import { CopilotConfirmation, CopilotError, CopilotReference, RequestDelta } from '../fetch'; +import { + FinishedCompletion, + SSEProcessor, + splitChunk, +} from '../stream'; + +suite('splitChunk', function () { + test('splits correctly with one newline in between', function () { + const [lines, extra] = splitChunk('foo\nbar'); + assert.deepStrictEqual(lines, ['foo']); + assert.strictEqual(extra, 'bar'); + }); + test('splits correctly with one newline in between and trailing', function () { + const [lines, extra] = splitChunk('foo\nbar\n'); + assert.deepStrictEqual(lines, ['foo', 'bar']); + assert.strictEqual(extra, ''); + }); + test('splits correctly with two newlines in between', function () { + const [lines, extra] = splitChunk('foo\n\nbar'); + assert.deepStrictEqual(lines, ['foo']); + assert.strictEqual(extra, 'bar'); + }); + test('splits correctly with two newlines in between and trailing', function () { + const [lines, extra] = splitChunk('foo\n\nbar\n\n'); + assert.deepStrictEqual(lines, ['foo', 'bar']); + assert.strictEqual(extra, ''); + }); + test('splits correctly with three newlines in between', function () { + const [lines, extra] = splitChunk('foo\n\n\nbar'); + assert.deepStrictEqual(lines, ['foo']); + assert.strictEqual(extra, 'bar'); + }); + test('splits correctly with three newlines in between and trailing', function () { + const [lines, extra] = splitChunk('foo\n\n\nbar\n\n\n'); + assert.deepStrictEqual(lines, ['foo', 'bar']); + assert.strictEqual(extra, ''); + }); +}); + +suite('Copilot Annotations', function () { + class TestCopilotAnnotation implements CopilotAnnotation { + id: number; + stop_offset: number; + start_offset: number; + details: { [key: string]: unknown }; + + constructor(id: number, start_offset: number, stop_offset: number, cursor: number) { + this.id = id; + this.start_offset = start_offset; + this.stop_offset = stop_offset; + this.details = { cursor: cursor }; + } + } + + test('add a new annotation', function () { + const annotations = new StreamCopilotAnnotations(); + const annotation = new TestCopilotAnnotation(1, 0, 1, 100); + annotations.update({ test: [annotation] }); + assert.deepStrictEqual(annotations.for('test'), [annotation]); + }); + + test('update many annotations', function () { + const annotations = new StreamCopilotAnnotations(); + const annotation = new TestCopilotAnnotation(1, 0, 1, 100); + const annotation2 = new TestCopilotAnnotation(2, 0, 1, 100); + const annotation3 = new TestCopilotAnnotation(1, 0, 1, 100); + const annotation4 = new TestCopilotAnnotation(2, 0, 1, 100); + annotations.update({ test: [annotation, annotation4], test2: [annotation2], test3: [annotation3] }); + assert.deepStrictEqual(annotations.for('test'), [annotation, annotation4]); + assert.deepStrictEqual(annotations.for('test2'), [annotation2]); + assert.deepStrictEqual(annotations.for('test3'), [annotation3]); + annotation.details['cursor'] = 102; + annotation2.details['cursor'] = 101; + annotation3.details['cursor'] = 103; + const annotation5 = new TestCopilotAnnotation(5, 0, 1, 103); + annotations.update({ test: [annotation], test2: [annotation2], test3: [annotation3, annotation5] }); + assert.deepStrictEqual(annotations.for('test'), [annotation, annotation4]); + assert.deepStrictEqual(annotations.for('test2'), [annotation2]); + assert.deepStrictEqual(annotations.for('test3'), [annotation3, annotation5]); + }); + + test('adds new annotation when new id started', function () { + const annotations = new StreamCopilotAnnotations(); + const annotation = new TestCopilotAnnotation(1, 0, 1, 100); + const annotation2 = new TestCopilotAnnotation(2, 0, 1, 100); + annotations.update({ test: [annotation] }); + annotations.update({ test: [annotation2] }); + assert.deepStrictEqual(annotations.for('test'), [annotation, annotation2]); + annotations.update({ test2: [annotation2] }); + assert.deepStrictEqual(annotations.for('test2'), [annotation2]); + }); +}); + +suite('SSEProcessor', function () { + let accessor: ServicesAccessor; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + }); + + interface SimpleResult { + finishReason: string | null; + chunks: string[]; + } + + function assertSimplifiedResultsEqual( + actual: FinishedCompletion[], + splifiedExpected: Record<number, SimpleResult> + ) { + const simplifiedActual = Object.fromEntries( + actual.map(c => [ + c.index, + { + finishReason: c.reason, + chunks: c.solution.text, + }, + ]) + ); + assert.deepStrictEqual(simplifiedActual, splifiedExpected); + } + + test('empty response yields no results', async function () { + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(''), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('done response yields no results', async function () { + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse('data: [DONE]\n'), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('broken JSON response is skipped', async function () { + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse('data: {\n'), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('empty JSON response is skipped', async function () { + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse('data: {}\n'), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('1 text token response yields 1 result', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":"stop"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'stop', + chunks: ['foo'], + }, + }); + }); + + test('does not fail with null choices', async function () { + const response = `data: {"choices":null} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('1 delta token response yields 1 result', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":"stop"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'stop', + chunks: ['foo'], + }, + }); + }); + + test('response with text and without finish_reason yields "DONE" result', async function () { + // This is not an expected case, since the OpenAI API should always + // include a finish_reason, but we handle it anyway. + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'DONE', + chunks: ['foo'], + }, + }); + }); + + test('response with delta and without finish_reason yields "DONE" result', async function () { + // This is not an expected case, since the OpenAI API should always + // include a finish_reason, but we handle it anyway. + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'DONE', + chunks: ['foo'], + }, + }); + }); + + test('response with text and without finish_reason or "[DONE]" yields "Iteration Done" result', async function () { + // This is not an expected case, since the OpenAI API should always + // include a finish_reason, but we handle it anyway. + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null}]} +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'Iteration Done', + chunks: ['foo'], + }, + }); + }); + + test('response with delta and without finish_reason or "[DONE]" yields "Iteration Done" result', async function () { + // This is not an expected case, since the OpenAI API should always + // include a finish_reason, but we handle it anyway. + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null}]} +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'Iteration Done', + chunks: ['foo'], + }, + }); + }); + + test('2 token text response with 1 index yields 1 result', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null}]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"stop"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'stop', + chunks: ['foo', 'bar'], + }, + }); + }); + + test('2 token delta response with 1 index yields 1 result', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null}]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"stop"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'stop', + chunks: ['foo', 'bar'], + }, + }); + }); + + test('text choice with logprobs are preserved', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null,"logprobs":{"token_logprobs":[-1.0]}}]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"stop","logprobs":{"token_logprobs":[-2.0]}}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assert.deepStrictEqual(results[0].solution.logprobs, [[-1], [-2]]); + }); + + test('delta choice with logprobs are preserved', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null,"logprobs":{"token_logprobs":[-1.0]}}]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"stop","logprobs":{"token_logprobs":[-2.0]}}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assert.deepStrictEqual(results[0].solution.logprobs, [[-1], [-2]]); + }); + + test('text choice with annotations are preserved', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null,"copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } }]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"stop","logprobs":{"token_logprobs":[-2.0]}}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const match = { match_id: 2, cursor: '123,', start_offset: 120, stop_offset: 130 }; + const results = await asyncIterableToArray(processor.processSSE()); + assert.deepStrictEqual(results[0].solution.copilot_annotations.for('code_references')[0], match); + }); + + test('delta choice with annotations are preserved', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null, "annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } }]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"stop", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } }]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + const match = { match_id: 2, cursor: '123,', start_offset: 120, stop_offset: 130 }; + assert.deepStrictEqual(results[0].solution.copilot_annotations.for('code_references')[0], match); + }); + + test('text choice with annotations are updated', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null,"copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } }]} +data: {"choices":[{"text":"bar","index":0,"copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,456", "start_offset": 120, "stop_offset": 140}] },"finish_reason":"stop","logprobs":{"token_logprobs":[-2.0]}}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const match = { match_id: 2, cursor: '123,456', start_offset: 120, stop_offset: 140 }; + const results = await asyncIterableToArray(processor.processSSE()); + assert.deepStrictEqual(results[0].solution.copilot_annotations.for('code_references')[0], match); + }); + + test('delta choice with annotations are updated', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":null }]} +data: {"choices":[{"delta":{"content":"bar", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":"stop" }]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const match = { match_id: 2, cursor: '123', start_offset: 120, stop_offset: 130 }; + const results = await asyncIterableToArray(processor.processSSE()); + assert.deepStrictEqual(results[0].solution.copilot_annotations.for('code_references')[0], match); + }); + + test('2 text token response with 2 indexes yields 2 results', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":"stop"}]} +data: {"choices":[{"text":"bar","index":1,"finish_reason":"stop"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 2, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'stop', + chunks: ['foo'], + }, + 1: { + finishReason: 'stop', + chunks: ['bar'], + }, + }); + }); + + test('2 delta token response with 2 indexes yields 2 results', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":"stop"}]} +data: {"choices":[{"delta":{"content":"bar"},"index":1,"finish_reason":"stop"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 2, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'stop', + chunks: ['foo'], + }, + 1: { + finishReason: 'stop', + chunks: ['bar'], + }, + }); + }); + + test('text completions that finish with "content_filter" are fully skipped when drop completion reasons are specified', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null}]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"content_filter"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting(), + ['content_filter'] + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('delta completions that finish with "content_filter" are fully skipped when drop completion reasons are specified', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null}]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"content_filter"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting(), + ['content_filter'] + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, {}); + }); + + test('text completions that finish with "content_filter" are returned, when drop completion reasons are empty', async function () { + const response = `data: {"choices":[{"text":"foo","index":0,"finish_reason":null}]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"content_filter"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting(), + [] + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'content_filter', + chunks: ['foo', 'bar'], + }, + }); + }); + + test('delta completions that finish with "content_filter" are returned, when drop completion reasons are empty', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo"},"index":0,"finish_reason":null}]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"content_filter"}]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting(), + [] + ); + const results = await asyncIterableToArray(processor.processSSE()); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: 'content_filter', + chunks: ['foo', 'bar'], + }, + }); + }); + + test('annotations are passed to finishedCb', async function () { + const references: CopilotAnnotation[] = []; + const response = `data: {"choices":[{"delta":{"content":"foo", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":null }]} +data: {"choices":[{"delta":{"content":"bar", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":"stop" }]} +data: [DONE] +`; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + + await asyncIterableToArray( + processor.processSSE((text: string, delta: RequestDelta) => { + delta.annotations?.for('code_references').forEach(ref => references.push(ref)); + return 0; + }) + ); + + const match = { match_id: 2, cursor: '123', start_offset: 120, stop_offset: 130 }; + assert.deepStrictEqual(references[0], match); + }); + + test('copilot_errors are passed to finishedCb', async function () { + const errors: CopilotError[] = []; + const response = `data: {"choices":[{"delta":{"content":"foo", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":null }]} +data: {"choices":[{"delta":{"content":"bar", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":"stop" }]} +data: {"copilot_errors": [{ "type": "reference", "code": "unknown", "message": "Unknown branch", "identifier": "id1" }, { "type": "reference", "code": "invalid", "message": "Invalid SHA", "identifier": "id2" }]} +data: [DONE] +`; + + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + + await asyncIterableToArray( + processor.processSSE((text: string, delta: RequestDelta) => { + delta.copilotErrors?.forEach(err => errors.push(err)); + return 0; + }) + ); + + assert.deepStrictEqual(errors.length, 2); + assert.deepStrictEqual(errors[0], { + type: 'reference', + code: 'unknown', + message: 'Unknown branch', + identifier: 'id1', + }); + assert.deepStrictEqual(errors[1], { + type: 'reference', + code: 'invalid', + message: 'Invalid SHA', + identifier: 'id2', + }); + }); + + test('copilot_confirmations are passed to finishedCb', async function () { + const confirmations: CopilotConfirmation[] = []; + const response = `data: {"choices":[{"delta":{"content":"foo", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123,", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":null }]} +data: {"choices":[{"delta":{"content":"bar", "copilot_annotations": {"code_references": [{"match_id": 2, "cursor": "123", "start_offset": 120, "stop_offset": 130}] } },"index":0,"finish_reason":"stop" }]} +data: {"choices":null,"copilot_confirmation":{"type":"action","title":"Are you sure you want to proceed?","message":"This action is irreversible.","confirmation":{"id":"123"}},"id":null} +data: [DONE] +`; + + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + + await asyncIterableToArray( + processor.processSSE((text: string, delta: RequestDelta) => { + if (delta.copilotConfirmation) { + confirmations.push(delta.copilotConfirmation); + } + return 0; + }) + ); + + assert.deepStrictEqual(confirmations.length, 1); + assert.deepStrictEqual(confirmations[0], { + type: 'action', + title: 'Are you sure you want to proceed?', + message: 'This action is irreversible.', + confirmation: { id: '123' }, + }); + }); + + test('n=1 text completion is truncated with finishedCb', async function () { + const response = `data: {"choices":[{"text":"foo\\n","index":0,"finish_reason":null}]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"stop"}]} +data: [DONE] + `; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE(() => 0)); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: null, + chunks: ['foo\n'], + }, + }); + }); + + test('n=1 delta completion is truncated with finishedCb', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo\\n"},"index":0,"finish_reason":null}]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"stop"}]} +data: [DONE] + `; + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE(() => 0)); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: null, + chunks: ['foo\n'], + }, + }); + }); + + test('n=2 text completion is truncated with finishedCb', async function () { + const response = `data: {"choices":[{"text":"foo\\n","index":0,"finish_reason":null}]} +data: {"choices":[{"text":"baz\\n","index":1,"finish_reason":null}]} +data: {"choices":[{"text":"bar","index":0,"finish_reason":"stop"}]} +data: {"choices":[{"text":"quux","index":1,"finish_reason":"stop"}]} +data: [DONE] + `; + const processor = await SSEProcessor.create( + accessor, + 2, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE(() => 0)); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: null, + chunks: ['foo\n'], + }, + 1: { + finishReason: null, + chunks: ['baz\n'], + }, + }); + }); + + test('n=2 delta completion is truncated with finishedCb', async function () { + const response = `data: {"choices":[{"delta":{"content":"foo\\n"},"index":0,"finish_reason":null}]} +data: {"choices":[{"delta":{"content":"baz\\n"},"index":1,"finish_reason":null}]} +data: {"choices":[{"delta":{"content":"bar"},"index":0,"finish_reason":"stop"}]} +data: {"choices":[{"delta":{"content":"quux"},"index":1,"finish_reason":"stop"}]} +data: [DONE] + `; + const processor = await SSEProcessor.create( + accessor, + 2, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + const results = await asyncIterableToArray(processor.processSSE(() => 0)); + assertSimplifiedResultsEqual(results, { + 0: { + finishReason: null, + chunks: ['foo\n'], + }, + 1: { + finishReason: null, + chunks: ['baz\n'], + }, + }); + }); + + test('copilot references', async function () { + const references: CopilotReference[] = []; + const response = `data: {"choices":[{"delta":{"content":"[{\\"type\\":\\"github.web-search\\",\\"data\\":{\\"query\\":\\"most recent version of React\\",\\"results\\":[{\\"title\\":\\"React v18.0 รƒยขร‚โ‚ฌร‚โ€œ React\\",\\"excerpt\\":\\"React v18.0. March 29, 2022 by The React Team. React 18 is now available on npm! In our last post, we shared step-by-step instructions for upgrading your app to React 18. In this post, weรƒยขร‚โ‚ฌร‚โ„ขll give an overview of whatรƒยขร‚โ‚ฌร‚โ„ขs new in React 18, and what it means for the future. Our latest major version includes out-of-the-box improvements like ...\\",\\"url\\":\\"https://react.dev/blog/2022/03/29/react-v18\\"},{\\"title\\":\\"React Versions รƒยขร‚โ‚ฌร‚โ€œ React\\",\\"excerpt\\":\\"React Versions. The React docs at react.dev provide documentation for the latest version of React. We aim to keep the docs updated within major versions, and do not publish versions for each minor or patch version. When a new major is released, we archive the docs for the previous version as x.react.dev. See our versioning policy for more info.\\",\\"url\\":\\"https://react.dev/versions\\"},{\\"title\\":\\"React 19 RC รƒยขร‚โ‚ฌร‚โ€œ React\\",\\"excerpt\\":\\"April 25, 2024 by The React Team. React 19 RC is now available on npm! In our React 19 RC Upgrade Guide, we shared step-by-step instructions for upgrading your app to React 19. In this post, weรƒยขร‚โ‚ฌร‚โ„ขll give an overview of the new features in React 19, and how you can adopt them. Whatรƒยขร‚โ‚ฌร‚โ„ขs new in React 19. Improvements in React 19.\\",\\"url\\":\\"https://react.dev/blog/2024/04/25/react-19\\"},{\\"title\\":\\"React 18: A Comprehensive Guide to the Latest Features and ... - Medium\\",\\"excerpt\\":\\"Lets explore the most recent version of React, diving into key features, improvements, and best practices to leverage in your projects. Hey fellow developer! Welcome to this comprehensive guide onรƒยขร‚โ‚ฌร‚ยฆ\\",\\"url\\":\\"https://medium.com/@vyakymenko/react-18-a-comprehensive-guide-to-the-latest-features-and-improvements-82825f209ae7\\"},{\\"title\\":\\"React\\",\\"excerpt\\":\\"React is designed to let you seamlessly combine components written by independent people, teams, and organizations. ... Latest React News. React Conf 2024 Recap. May 22, 2024. React 19 RC. April 25, 2024. React 19 RC Upgrade Guide. April 25, 2024. React Labs: February 2024. February 15, 2024.\\",\\"url\\":\\"https://19.react.dev/\\"}],\\"type\\":\\"web-search\\"},\\"id\\":\\"web-search: most recent version of React\\",\\"metadata\\":{\\"display_name\\":\\"web-search: most recent version of React\\",\\"display_icon\\":\\"\\"}}]","name":"bing-search","role":"function"},"index":0}],"copilot_references":[{"type":"github.web-search","data":{"query":"most recent version of React","results":[{"title":"React v18.0 รƒยขร‚โ‚ฌร‚โ€œ React","excerpt":"React v18.0. March 29, 2022 by The React Team. React 18 is now available on npm! In our last post, we shared step-by-step instructions for upgrading your app to React 18. In this post, weรƒยขร‚โ‚ฌร‚โ„ขll give an overview of whatรƒยขร‚โ‚ฌร‚โ„ขs new in React 18, and what it means for the future. Our latest major version includes out-of-the-box improvements like ...","url":"https://react.dev/blog/2022/03/29/react-v18"},{"title":"React Versions รƒยขร‚โ‚ฌร‚โ€œ React","excerpt":"React Versions. The React docs at react.dev provide documentation for the latest version of React. We aim to keep the docs updated within major versions, and do not publish versions for each minor or patch version. When a new major is released, we archive the docs for the previous version as x.react.dev. See our versioning policy for more info.","url":"https://react.dev/versions"},{"title":"React 19 RC รƒยขร‚โ‚ฌร‚โ€œ React","excerpt":"April 25, 2024 by The React Team. React 19 RC is now available on npm! In our React 19 RC Upgrade Guide, we shared step-by-step instructions for upgrading your app to React 19. In this post, weรƒยขร‚โ‚ฌร‚โ„ขll give an overview of the new features in React 19, and how you can adopt them. Whatรƒยขร‚โ‚ฌร‚โ„ขs new in React 19. Improvements in React 19.","url":"https://react.dev/blog/2024/04/25/react-19"},{"title":"React 18: A Comprehensive Guide to the Latest Features and ... - Medium","excerpt":"Lets explore the most recent version of React, diving into key features, improvements, and best practices to leverage in your projects. Hey fellow developer! Welcome to this comprehensive guide onรƒยขร‚โ‚ฌร‚ยฆ","url":"https://medium.com/@vyakymenko/react-18-a-comprehensive-guide-to-the-latest-features-and-improvements-82825f209ae7"},{"title":"React","excerpt":"React is designed to let you seamlessly combine components written by independent people, teams, and organizations. ... Latest React News. React Conf 2024 Recap. May 22, 2024. React 19 RC. April 25, 2024. React 19 RC Upgrade Guide. April 25, 2024. React Labs: February 2024. February 15, 2024.","url":"https://19.react.dev/"}],"type":"web-search"},"id":"web-search: most recent version of React","metadata":{"display_name":"web-search: most recent version of React","display_icon":""}}],"id":null} +data: [DONE] +`; + + const processor = await SSEProcessor.create( + accessor, + 1, + createFakeStreamResponse(response), + TelemetryWithExp.createEmptyConfigForTesting() + ); + + await asyncIterableToArray( + processor.processSSE((text: string, delta: RequestDelta) => { + delta.copilotReferences?.forEach(ref => references.push(ref)); + return 0; + }) + ); + + assert.deepStrictEqual(references.length, 1); + assert.deepStrictEqual(references[0], { + type: 'github.web-search', + data: { + query: 'most recent version of React', + results: [ + { + title: 'React v18.0 รƒยขร‚โ‚ฌร‚โ€œ React', + excerpt: + 'React v18.0. March 29, 2022 by The React Team. React 18 is now available on npm! In our last post, we shared step-by-step instructions for upgrading your app to React 18. In this post, weรƒยขร‚โ‚ฌร‚โ„ขll give an overview of whatรƒยขร‚โ‚ฌร‚โ„ขs new in React 18, and what it means for the future. Our latest major version includes out-of-the-box improvements like ...', + url: 'https://react.dev/blog/2022/03/29/react-v18', + }, + { + title: 'React Versions รƒยขร‚โ‚ฌร‚โ€œ React', + excerpt: + 'React Versions. The React docs at react.dev provide documentation for the latest version of React. We aim to keep the docs updated within major versions, and do not publish versions for each minor or patch version. When a new major is released, we archive the docs for the previous version as x.react.dev. See our versioning policy for more info.', + url: 'https://react.dev/versions', + }, + { + title: 'React 19 RC รƒยขร‚โ‚ฌร‚โ€œ React', + excerpt: + 'April 25, 2024 by The React Team. React 19 RC is now available on npm! In our React 19 RC Upgrade Guide, we shared step-by-step instructions for upgrading your app to React 19. In this post, weรƒยขร‚โ‚ฌร‚โ„ขll give an overview of the new features in React 19, and how you can adopt them. Whatรƒยขร‚โ‚ฌร‚โ„ขs new in React 19. Improvements in React 19.', + url: 'https://react.dev/blog/2024/04/25/react-19', + }, + { + title: 'React 18: A Comprehensive Guide to the Latest Features and ... - Medium', + excerpt: + 'Lets explore the most recent version of React, diving into key features, improvements, and best practices to leverage in your projects. Hey fellow developer! Welcome to this comprehensive guide onรƒยขร‚โ‚ฌร‚ยฆ', + url: 'https://medium.com/@vyakymenko/react-18-a-comprehensive-guide-to-the-latest-features-and-improvements-82825f209ae7', + }, + { + title: 'React', + excerpt: + 'React is designed to let you seamlessly combine components written by independent people, teams, and organizations. ... Latest React News. React Conf 2024 Recap. May 22, 2024. React 19 RC. April 25, 2024. React 19 RC Upgrade Guide. April 25, 2024. React Labs: February 2024. February 15, 2024.', + url: 'https://19.react.dev/', + }, + ], + type: 'web-search', + }, + id: 'web-search: most recent version of React', + metadata: { + display_name: 'web-search: most recent version of React', + display_icon: '', + }, + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/postInsertion.ts b/completions-sample-code/vscode-node/lib/src/postInsertion.ts new file mode 100644 index 0000000..e53f177 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/postInsertion.ts @@ -0,0 +1,471 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { CopilotNamedAnnotationList } from '../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTelemetryService } from '../../bridge/src/completionsTelemetryServiceBridge'; +import { ICompletionsCopilotTokenManager } from './auth/copilotTokenManager'; +import { ChangeTracker } from './changeTracker'; +import { ICompletionsCitationManager, IPCitationDetail } from './citationManager'; +import { createCompletionState } from './completionState'; +import { ICompletionsFileReaderService } from './fileReader'; +import { PostInsertionCategory, telemetryAccepted, telemetryRejected } from './ghostText/telemetry'; +import { ICompletionsLogTargetService, Logger } from './logger'; +import { contextIndentationFromText, indentationBlockFinished } from './prompt/parseBlock'; +import { Prompt, extractPrompt } from './prompt/prompt'; +import { fetchCitations } from './snippy/handlePostInsertion'; +import { editDistance, lexEditDistance } from './suggestions/editDistance'; +import { SuggestionStatus, computeCompletionText } from './suggestions/partialSuggestions'; +import { TelemetryStore, TelemetryWithExp, telemetry, telemetryCatch } from './telemetry'; +import { ICompletionsTextDocumentManagerService } from './textDocumentManager'; +import { ICompletionsPromiseQueueService } from './util/promiseQueue'; +import { ICompletionsRuntimeModeService } from './util/runtimeMode'; + +const postInsertionLogger = new Logger('postInsertion'); + +type Timeout = { + seconds: number; + captureCode: boolean; + captureRejection: boolean; +}; +// windows for telemetry checks, in seconds +// captureCode = capture the code after acceptance, +// captureRejection = capture the code after rejection +const captureTimeouts: Timeout[] = [ + { seconds: 15, captureCode: false, captureRejection: false }, + { seconds: 30, captureCode: true, captureRejection: true }, + { seconds: 120, captureCode: false, captureRejection: false }, + { seconds: 300, captureCode: false, captureRejection: false }, + { seconds: 600, captureCode: false, captureRejection: false }, +]; + +// No. of chars before/after insertion point to look for the completion +const stillInCodeNearMargin = 50; +const stillInCodeFarMargin = 1500; + +// If lex edit distance is below this fraction of completion length it is considered +// in the code +const stillInCodeFraction = 0.5; + +// Number of characters captured after the insertion point. +// Used only if we couldn't detect termination point with indent-based parsing. +const captureCodeMargin = 500; + +const postInsertConfiguration: { + triggerPostInsertionSynchroneously: boolean; + captureCode: boolean; + captureRejection: boolean; +} = { + triggerPostInsertionSynchroneously: false, + captureCode: false, + captureRejection: false, +}; + +async function captureCode( + accessor: ServicesAccessor, + uri: string, + completionTelemetry: TelemetryWithExp, + offset: number, + suffixOffset?: number +): Promise<{ prompt: Prompt; capturedCode: string; terminationOffset: number }> { + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const result = await accessor.get(ICompletionsFileReaderService).getOrReadTextDocumentWithFakeClientProperties({ uri }); + if (result.status !== 'valid') { + postInsertionLogger.info(logTarget, `Could not get document for ${uri}. Maybe it was closed by the editor.`); + return { + prompt: { + prefix: '', + suffix: '', + isFimEnabled: false, + }, + capturedCode: '', + terminationOffset: 0, + }; + } + const document = result.document; + const documentText = document.getText(); + const documentTextBefore = documentText.substring(0, offset); + const position = document.positionAt(offset); + + // Treat the code before offset as the hypothetical prompt + const hypotheticalPromptResponse = await instantiationService.invokeFunction(extractPrompt, + completionTelemetry.properties.headerRequestId, + createCompletionState(document, position), + completionTelemetry + ); + const hypotheticalPrompt = + hypotheticalPromptResponse.type === 'prompt' + ? hypotheticalPromptResponse.prompt + : { + prefix: documentTextBefore, + suffix: '', + isFimEnabled: false, + }; // TODO(eaftan): Pass an actual suffix when we're ready to support it + + if (hypotheticalPrompt.isFimEnabled && suffixOffset !== undefined) { + // With FIM enabled, we can exactly determine capturedCode, suffix and prefix by propertly initialized trackers. No need to guess. + const capturedCode = documentText.substring(offset, suffixOffset); + hypotheticalPrompt.suffix = documentText.substring(suffixOffset); + + return { prompt: hypotheticalPrompt, capturedCode, terminationOffset: 0 }; + } else { + //Everything after the insertion point is hypothetical response we could get from AI + const hypotheticalResponse = documentText.substring(offset); + + //Try to find the termination offset in the hypothetical response using indentation based parsing + const contextIndent = contextIndentationFromText(documentTextBefore, offset, document.detectedLanguageId); + const indentTerminationFunction = indentationBlockFinished(contextIndent, undefined); + const terminationResult = indentTerminationFunction(hypotheticalResponse); + + //If we could detect termination of the indentation block, capture 2x length of detected suggestion + //Otherwise capture a lot of characters + const maxOffset = Math.min( + documentText.length, + offset + (terminationResult ? terminationResult * 2 : captureCodeMargin) + ); + + const capturedCode = documentText.substring(offset, maxOffset); + + return { prompt: hypotheticalPrompt, capturedCode, terminationOffset: terminationResult ?? -1 }; + } +} + +export function postRejectionTasks( + accessor: ServicesAccessor, + insertionCategory: PostInsertionCategory, + insertionOffset: number, + uri: string, + completions: { completionText: string; completionTelemetryData: TelemetryWithExp }[] +) { + const logTarget = accessor.get(ICompletionsLogTargetService); + const instantiationService = accessor.get(IInstantiationService); + const telemetryService = accessor.get(ICompletionsTelemetryService); + const promiseQueueService = accessor.get(ICompletionsPromiseQueueService); + + //Send `.rejected` telemetry event for each rejected completion + completions.forEach(({ completionText, completionTelemetryData }) => { + postInsertionLogger.debug( + logTarget, + `${insertionCategory}.rejected choiceIndex: ${completionTelemetryData.properties.choiceIndex}` + ); + instantiationService.invokeFunction(telemetryRejected, insertionCategory, completionTelemetryData); + }); + const positionTracker = instantiationService.createInstance(ChangeTracker, uri, insertionOffset - 1); + const suffixTracker = instantiationService.createInstance(ChangeTracker, uri, insertionOffset); + + const checkInCode = async (t: Timeout) => { + postInsertionLogger.debug( + logTarget, + `Original offset: ${insertionOffset}, Tracked offset: ${positionTracker.offset}` + ); + const { completionTelemetryData } = completions[0]; + + const { prompt, capturedCode, terminationOffset } = await instantiationService.invokeFunction(captureCode, + uri, + completionTelemetryData, + positionTracker.offset + 1, + suffixTracker.offset + ); + + const promptTelemetry = { + hypotheticalPromptJson: JSON.stringify({ prefix: prompt.prefix, context: prompt.context }), + hypotheticalPromptSuffixJson: JSON.stringify(prompt.suffix), + }; + + const customTelemetryData = completionTelemetryData.extendedBy( + { + ...promptTelemetry, + capturedCodeJson: JSON.stringify(capturedCode), + }, + { + timeout: t.seconds, + insertionOffset: insertionOffset, + trackedOffset: positionTracker.offset, + terminationOffsetInCapturedCode: terminationOffset, + } + ); + postInsertionLogger.debug( + logTarget, + `${insertionCategory}.capturedAfterRejected choiceIndex: ${completionTelemetryData.properties.choiceIndex}`, + customTelemetryData + ); + instantiationService.invokeFunction(telemetry, insertionCategory + '.capturedAfterRejected', customTelemetryData, TelemetryStore.Enhanced); + }; + // Capture the code typed after we detected that completion was rejected, + // Uses first displayed completion as the source/seed of telemetry information. + captureTimeouts + .filter(t => t.captureRejection) + .map(t => + positionTracker.push( + telemetryCatch(telemetryService, promiseQueueService, () => checkInCode(t), 'postRejectionTasks'), + t.seconds * 1000 + ) + ); +} + +export function postInsertionTasks( + accessor: ServicesAccessor, + insertionCategory: PostInsertionCategory, + completionText: string, + insertionOffset: number, + uri: string, + telemetryData: TelemetryWithExp, + suggestionStatus: SuggestionStatus, + copilotAnnotations?: CopilotNamedAnnotationList +) { + const logTarget = accessor.get(ICompletionsLogTargetService); + const instantiationService = accessor.get(IInstantiationService); + const promiseQueueService = accessor.get(ICompletionsPromiseQueueService); + const telemetryService = accessor.get(ICompletionsTelemetryService); + const runtimeModeService = accessor.get(ICompletionsRuntimeModeService); + + const telemetryDataWithStatus = telemetryData.extendedBy( + { + compType: suggestionStatus.compType, + }, + { + compCharLen: suggestionStatus.acceptedLength, + numLines: suggestionStatus.acceptedLines, + } + ); + // send ".accepted" telemetry + postInsertionLogger.debug( + logTarget, + `${insertionCategory}.accepted choiceIndex: ${telemetryDataWithStatus.properties.choiceIndex}` + ); + instantiationService.invokeFunction(telemetryAccepted, insertionCategory, telemetryDataWithStatus); + + const fullCompletionText = completionText; + completionText = computeCompletionText(completionText, suggestionStatus); + const trimmedCompletion = completionText.trim(); + const tracker = instantiationService.createInstance(ChangeTracker, uri, insertionOffset); + const suffixTracker = instantiationService.createInstance(ChangeTracker, uri, insertionOffset + completionText.length); + + const stillInCodeCheck = async (timeout: Timeout) => { + const check = instantiationService.invokeFunction(checkStillInCode, + insertionCategory, + trimmedCompletion, + insertionOffset, + uri, + timeout, + telemetryDataWithStatus, + tracker, + suffixTracker + ); + await check; + }; + + // For test purposes, we add one set of these telemetry events synchronously to allow asserting the telemetry + if (postInsertConfiguration.triggerPostInsertionSynchroneously && runtimeModeService.isRunningInTest()) { + const check = stillInCodeCheck({ + seconds: 0, + captureCode: postInsertConfiguration.captureCode, + captureRejection: postInsertConfiguration.captureRejection, + }); + promiseQueueService.register(check); + } else { + captureTimeouts.map(timeout => + tracker.push( + telemetryCatch(telemetryService, promiseQueueService, () => stillInCodeCheck(timeout), 'postInsertionTasks'), + timeout.seconds * 1000 + ) + ); + } + + instantiationService.invokeFunction(acc => telemetryCatch(telemetryService, promiseQueueService, citationCheck, 'post insertion citation check')( + acc, + uri, + fullCompletionText, + completionText, + insertionOffset, + copilotAnnotations + )); +} + +async function citationCheck( + accessor: ServicesAccessor, + uri: string, + fullCompletionText: string, + insertedText: string, + insertionOffset: number, + copilotAnnotations?: CopilotNamedAnnotationList +) { + const logTarget = accessor.get(ICompletionsLogTargetService); + const textDocumentManagerService = accessor.get(ICompletionsTextDocumentManagerService); + const copilotTokenManager = accessor.get(ICompletionsCopilotTokenManager); + const citationManagerService = accessor.get(ICompletionsCitationManager); + + // If there are no citations, request directly from the snippy service + if (!copilotAnnotations || (copilotAnnotations.ip_code_citations?.length ?? 0) < 1) { + // Do not request citations if in blocking mode + if (copilotTokenManager.getLastToken()?.getTokenValue('sn') === '1') { return; } + await fetchCitations(accessor, uri, insertedText, insertionOffset); + return; + } + + const doc = await textDocumentManagerService.getTextDocument({ uri }); + + // in the CLS, if the editor does not wait to send document updates until the + // acceptance function returns, we could be in a race condition with ongoing + // edits. This searches for the completion text so that hopefully we're providing + // an exact location in a known version of the document. + if (doc) { + const found = find(doc.getText(), insertedText, stillInCodeNearMargin, insertionOffset); + if (found.stillInCodeHeuristic) { + insertionOffset = found.foundOffset; + } + } + + for (const citation of copilotAnnotations.ip_code_citations) { + const citationStart = computeCitationStart( + fullCompletionText.length, + insertedText.length, + citation.start_offset + ); + if (citationStart === undefined) { + postInsertionLogger.info( + logTarget, + `Full completion for ${uri} contains a reference matching public code, but the partially inserted text did not include the match.` + ); + continue; + } + const offsetStart = insertionOffset + citationStart; + const start = doc?.positionAt(offsetStart); + const offsetEnd = + insertionOffset + computeCitationEnd(fullCompletionText.length, insertedText.length, citation.stop_offset); + const end = doc?.positionAt(offsetEnd); + const text = start && end ? doc?.getText({ start, end }) : '<unknown>'; + + await citationManagerService.handleIPCodeCitation({ + inDocumentUri: uri, + offsetStart, + offsetEnd, + version: doc?.version, + location: start && end ? { start, end } : undefined, + matchingText: text, + details: citation.details.citations as IPCitationDetail[], + }); + } +} + +function computeCitationStart( + completionLength: number, + insertedLength: number, + citationStartOffset: number +): number | undefined { + if (insertedLength < completionLength && citationStartOffset > insertedLength) { + return undefined; + } + return citationStartOffset; +} + +function computeCitationEnd(completionLength: number, insertedLength: number, citationStopOffset: number): number { + if (insertedLength < completionLength) { + return Math.min(citationStopOffset, insertedLength); + } + return citationStopOffset; +} + +function find(documentText: string, completion: string, margin: number, offset: number) { + // Compute the best alignment between a window of the document text and the completion + const window = documentText.substring( + Math.max(0, offset - margin), + Math.min(documentText.length, offset + completion.length + margin) + ); + const lexAlignment = lexEditDistance(window, completion); + const fraction = lexAlignment.lexDistance / lexAlignment.needleLexLength; + const { distance: charEditDistance } = editDistance( + window.substring(lexAlignment.startOffset, lexAlignment.endOffset), + completion + ); + return { + relativeLexEditDistance: fraction, + charEditDistance, + completionLexLength: lexAlignment.needleLexLength, + foundOffset: lexAlignment.startOffset + Math.max(0, offset - margin), + lexEditDistance: lexAlignment.lexDistance, + stillInCodeHeuristic: fraction <= stillInCodeFraction ? 1 : 0, + }; +} + +async function checkStillInCode( + accessor: ServicesAccessor, + insertionCategory: string, + completion: string, + insertionOffset: number, // offset where the completion was inserted to + uri: string, + timeout: Timeout, + telemetryData: TelemetryWithExp, + tracker: ChangeTracker, + suffixTracker: ChangeTracker +) { + // Get contents of file from file system + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const result = await accessor.get(ICompletionsFileReaderService).getOrReadTextDocument({ uri }); + if (result.status === 'valid') { + const document = result.document; + const documentText = document.getText(); + + // We try twice, first very close to the insertion point, then a bit + // further. This is to increase accuracy for short completions, + // where the completion might appear elsewhere. + let finding = find(documentText, completion, stillInCodeNearMargin, tracker.offset); + if (!finding.stillInCodeHeuristic) { + finding = find(documentText, completion, stillInCodeFarMargin, tracker.offset); + } + // Debug and log a binary decision + postInsertionLogger.debug( + logTarget, + `stillInCode: ${finding.stillInCodeHeuristic ? 'Found' : 'Not found'}! Completion '${completion}' in file ${uri + }. lexEditDistance fraction was ${finding.relativeLexEditDistance}. Char edit distance was ${finding.charEditDistance + }. Inserted at ${insertionOffset}, tracked at ${tracker.offset}, found at ${finding.foundOffset + }. choiceIndex: ${telemetryData.properties.choiceIndex}` + ); + // Log all the details for analysis + const customTelemetryData = telemetryData + .extendedBy({}, { timeout: timeout.seconds, insertionOffset: insertionOffset, trackedOffset: tracker.offset }) + .extendedBy({}, finding); + instantiationService.invokeFunction(telemetry, insertionCategory + '.stillInCode', customTelemetryData); + + if (timeout.captureCode) { + const { prompt, capturedCode, terminationOffset } = await instantiationService.invokeFunction( + captureCode, + uri, + customTelemetryData, + tracker.offset, + suffixTracker.offset + ); + const promptTelemetry = { + hypotheticalPromptJson: JSON.stringify({ prefix: prompt.prefix, context: prompt.context }), + hypotheticalPromptSuffixJson: JSON.stringify(prompt.suffix), + }; + + const afterAcceptedTelemetry = telemetryData.extendedBy( + { + ...promptTelemetry, + capturedCodeJson: JSON.stringify(capturedCode), + }, + { + timeout: timeout.seconds, + insertionOffset: insertionOffset, + trackedOffset: tracker.offset, + terminationOffsetInCapturedCode: terminationOffset, + } + ); + postInsertionLogger.debug( + logTarget, + `${insertionCategory}.capturedAfterAccepted choiceIndex: ${telemetryData.properties.choiceIndex}`, + customTelemetryData + ); + instantiationService.invokeFunction( + telemetry, + insertionCategory + '.capturedAfterAccepted', + afterAcceptedTelemetry, + TelemetryStore.Enhanced + ); + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/progress.ts b/completions-sample-code/vscode-node/lib/src/progress.ts new file mode 100644 index 0000000..26d8004 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/progress.ts @@ -0,0 +1,96 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { Command, StatusKind } from '../../types/src'; + +export interface StatusChangedEvent { + kind: StatusKind; + message?: string; + busy: boolean; + command?: Command; +} + +export const ICompletionsStatusReporter = createServiceIdentifier<ICompletionsStatusReporter>('ICompletionsStatusReporter'); +export interface ICompletionsStatusReporter { + readonly _serviceBrand: undefined; + + busy: boolean; + + withProgress<T>(callback: () => Promise<T>): Promise<T>; + + forceStatus(kind: StatusKind, message?: string, command?: Command): void; + forceNormal(): void; + setError(message: string, command?: Command): void; + setWarning(message: string): void; + setInactive(message: string): void; + clearInactive(): void; +} + +export abstract class StatusReporter implements ICompletionsStatusReporter { + declare _serviceBrand: undefined; + + #inProgressCount = 0; + #kind: StatusKind = 'Normal'; + #message: string | undefined; + #command: Command | undefined; + #startup = true; + + abstract didChange(event: StatusChangedEvent): void; + + get busy() { + return this.#inProgressCount > 0; + } + + withProgress<T>(callback: () => Promise<T>): Promise<T> { + if (this.#kind === 'Warning') { this.forceNormal(); } + if (this.#inProgressCount++ === 0) { this.#didChange(); } + return callback().finally(() => { + if (--this.#inProgressCount === 0) { this.#didChange(); } + }); + } + + forceStatus(kind: StatusKind, message?: string, command?: Command) { + if (this.#kind === kind && this.#message === message && !command && !this.#command && !this.#startup) { return; } + this.#kind = kind; + this.#message = message; + this.#command = command; + this.#startup = false; + this.#didChange(); + } + + forceNormal() { + if (this.#kind === 'Inactive') { return; } + this.forceStatus('Normal'); + } + + setError(message: string, command?: Command) { + this.forceStatus('Error', message, command); + } + + setWarning(message: string) { + if (this.#kind === 'Error') { return; } + this.forceStatus('Warning', message); + } + + setInactive(message: string) { + if (this.#kind === 'Error' || this.#kind === 'Warning') { return; } + this.forceStatus('Inactive', message); + } + + clearInactive() { + if (this.#kind !== 'Inactive') { return; } + this.forceStatus('Normal'); + } + + #didChange() { + const event = { kind: this.#kind, message: this.#message, busy: this.busy, command: this.#command }; + this.didChange(event); + } +} + +// Don't delete. Needed for tests that don't care about status changes +export class NoOpStatusReporter extends StatusReporter { + override didChange() { } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/asyncUtils.ts b/completions-sample-code/vscode-node/lib/src/prompt/asyncUtils.ts new file mode 100644 index 0000000..a0e4639 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/asyncUtils.ts @@ -0,0 +1,176 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { Disposable } from 'vscode'; +import { CancellationToken } from 'vscode-languageserver-protocol'; +import { ResolveOnTimeoutResult, ResolveResult } from '../../../types/src'; +import { Deferred } from '../util/async'; + +/** + * Converts an event to a Promise that resolves when the event is fired + * @param subscribe A function that takes a listener and returns a Disposable for cleanup + * @returns A Promise that resolves with the event data when the event fires + */ +export async function eventToPromise<T>(subscribe: (listener: (event: T) => void) => Disposable): Promise<T> { + const deferred = new Deferred<T>(); + const disposable = subscribe((event: T) => { + deferred.resolve(event); + disposable.dispose(); + }); + return deferred.promise; +} + +/** + * Converts a CancellationToken to a Promise that resolves when cancellation is requested + * @param token The CancellationToken to observe + * @returns A Promise that resolves when the token is canceled + */ +async function cancellationTokenToPromise(token: CancellationToken): Promise<void> { + if (token.isCancellationRequested) { return; } + const deferred = new Deferred<void>(); + const disposable = token.onCancellationRequested(() => { + deferred.resolve(); + disposable.dispose(); + }); + await deferred.promise; +} + +async function raceCancellation(promise: Promise<void>, token?: CancellationToken): Promise<void> { + if (token) { + const cancellationPromise = cancellationTokenToPromise(token); + await Promise.race([promise, cancellationPromise]); + } else { + await promise; + } +} + +// Workaround for https://github.com/microsoft/TypeScript/issues/17002 +export function isArrayOfT<T>(value: ResolveOnTimeoutResult<T> | undefined): value is readonly T[] { + return Array.isArray(value); +} + +type ResolvedItem<T> = + | { + status: 'full' | 'partial'; + resolutionTime: number; + value: T[]; + } + | { + status: 'none'; + resolutionTime: number; + value: null; + } + | { + status: 'error'; + resolutionTime: number; + reason: unknown; + }; + +/** + * Resolves concurrently all given promises or async iterables, returning a map of their results. + * + * Given a collection of either promises resolving to single elements, arrays or async iterables, + * this function will resolve them all to arrays and return a map of the results. + * If a cancellation token is provided, when it is triggered, the function will stop resolving + * and return the results collected so far, with the async iterables potentially returning partial results. + * + * @param resolvables A map of keys to promises or async iterables. + * @param cancellation An optional cancellation promise. + * @returns A promise that resolves to a map of the results. + */ +export async function resolveAll<K, T>( + resolvables: Map<K, ResolveResult<T>>, + cancellationToken?: CancellationToken +): Promise<Map<K, ResolvedItem<T>>> { + const results: Map<K, ResolvedItem<T>> = new Map(); + const promises: Promise<void>[] = []; + for (const [key, resolvable] of resolvables.entries()) { + const promise = (async () => { + const result = await resolve(resolvable, cancellationToken); + results.set(key, result); + })(); + promises.push(promise); + } + await Promise.allSettled(promises.values()); + return results; +} + +async function resolve<T>( + resolvable: ResolveResult<T>, + cancellationToken?: CancellationToken +): Promise<ResolvedItem<T>> { + let result: ResolvedItem<T>; + if (resolvable instanceof Promise) { + result = await resolvePromise(resolvable, cancellationToken); + } else { + result = await resolveIterable(resolvable, cancellationToken); + } + return result; +} + +/** Resolves a promise until cancelled, and possibly converts result to array + */ +async function resolvePromise<T>( + promise: Promise<ResolveOnTimeoutResult<T>>, + cancellationToken?: CancellationToken +): Promise<ResolvedItem<T>> { + const startTime = performance.now(); + let resolved: ResolvedItem<T> = { status: 'none', resolutionTime: 0, value: null }; + const collectPromise = (async () => { + try { + const result = await promise; + if (cancellationToken?.isCancellationRequested) { + return; + } + resolved = { status: 'full', resolutionTime: 0, value: isArrayOfT<T>(result) ? [...result] : [result] }; + } catch (e) { + if (cancellationToken?.isCancellationRequested) { + return; + } + resolved = { status: 'error', resolutionTime: 0, reason: e }; + } + })(); + await raceCancellation(collectPromise, cancellationToken); + resolved.resolutionTime = performance.now() - startTime; + return resolved; +} + +/** Resolves an async iterable until cancelled + */ +async function resolveIterable<T>( + iterable: AsyncIterable<T>, + cancellationToken?: CancellationToken +): Promise<ResolvedItem<T>> { + const startTime = performance.now(); + let resolved: ResolvedItem<T> = { status: 'none', resolutionTime: 0, value: null }; + const collectPromise = (async () => { + try { + for await (const item of iterable) { + if (cancellationToken?.isCancellationRequested) { + return; + } + if (resolved.status !== 'partial') { + resolved = { status: 'partial', resolutionTime: 0, value: [] }; + } + resolved.value.push(item); + } + if (!cancellationToken?.isCancellationRequested) { + if (resolved.status !== 'partial') { + resolved = { status: 'full', resolutionTime: 0, value: [] }; + } else { + resolved.status = 'full'; + } + } + } catch (e) { + if (cancellationToken?.isCancellationRequested) { + return; + } + resolved = { status: 'error', resolutionTime: 0, reason: e }; + } + })(); + await raceCancellation(collectPromise, cancellationToken); + resolved.resolutionTime = performance.now() - startTime; + return resolved; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/cascadingPromptFactory.ts b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/cascadingPromptFactory.ts new file mode 100644 index 0000000..f2fb9c8 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/cascadingPromptFactory.ts @@ -0,0 +1,268 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IIgnoreService } from '../../../../../../../platform/ignore/common/ignoreService'; +import { URI } from '../../../../../../../util/vs/base/common/uri'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTelemetryService } from '../../../../bridge/src/completionsTelemetryServiceBridge'; +import { ComponentStatistics, PromptMetadata } from '../../../../prompt/src/components/components'; +import { commentBlockAsSingles } from '../../../../prompt/src/languageMarker'; +import { PromptComponentAllocation, PromptComponentId } from '../../../../prompt/src/prompt'; +import { TokenizerName } from '../../../../prompt/src/tokenization'; +import { CancellationToken } from '../../../../types/src'; +import { CompletionState } from '../../completionState'; +import { ICompletionsFeaturesService } from '../../experiments/featuresService'; +import { ICompletionsLogTargetService, logger } from '../../logger'; +import { telemetryException, TelemetryWithExp } from '../../telemetry'; +import { TextDocumentContents } from '../../textDocument'; +import { ICompletionsContextProviderBridgeService } from '../components/contextProviderBridge'; +import { + renderWithMetadata, + type RenderedComponent, + type ValidatedContextItems, + type VirtualPromptComponent, +} from '../components/virtualComponent'; +import { + ContextProviderTelemetry, + matchContextItems, + ResolvedContextItem, + telemetrizeContextItems, + useContextProviderAPI, +} from '../contextProviderRegistry'; +import { getCodeSnippetsFromContextItems } from '../contextProviders/codeSnippets'; +import { CodeSnippetWithId, TraitWithId } from '../contextProviders/contextItemSchemas'; +import { getTraitsFromContextItems, ReportTraitsTelemetry } from '../contextProviders/traits'; +import { componentStatisticsToPromptMatcher, ICompletionsContextProviderService } from '../contextProviderStatistics'; +import { + _contextTooShort, + _copilotContentExclusion, + _promptCancelled, + _promptError, + getPromptOptions, + MIN_PROMPT_CHARS, + PromptResponse, + trimLastLine, +} from '../prompt'; +import { + CompletionsPromptOptions, + ICompletionsPromptFactoryService +} from './completionsPromptFactory'; + +// If the space allocated to the suffix is at least this fraction of the estimated suffix cost, +// we will render the suffix before the prefix and use any surplus suffix budget to fill the prefix. +// Otherwise, we render the prefix first and use any surplus prefix budget to fill the suffix. +const SMALL_SUFFIX_THRESHOLD = 0.8; + +export abstract class CascadingPromptFactory implements ICompletionsPromptFactoryService { + declare _serviceBrand: undefined; + private renderId = 0; + + constructor( + protected components: Record<PromptComponentId, VirtualPromptComponent>, + @IIgnoreService protected readonly ignoreService: IIgnoreService, + @IInstantiationService protected readonly instantiationService: IInstantiationService, + @ICompletionsFeaturesService protected readonly featuresService: ICompletionsFeaturesService, + @ICompletionsTelemetryService protected readonly completionsTelemetryService: ICompletionsTelemetryService, + @ICompletionsContextProviderBridgeService protected readonly contextProviderBridge: ICompletionsContextProviderBridgeService, + @ICompletionsLogTargetService protected readonly logTargetService: ICompletionsLogTargetService, + @ICompletionsContextProviderService protected readonly contextProviderStatistics: ICompletionsContextProviderService, + ) { } + + async prompt(opts: CompletionsPromptOptions, cancellationToken?: CancellationToken): Promise<PromptResponse> { + try { + return await this.createPromptUnsafe(opts, cancellationToken); + } catch (e) { + return this.errorPrompt(e as Error); + } + } + + getComponentAllocation(telemetryData: TelemetryWithExp): PromptComponentAllocation { + const suffixPercent = this.featuresService.suffixPercent(telemetryData); + const stableContextPercent = this.featuresService.stableContextPercent(telemetryData); + const volatileContextPercent = this.featuresService.volatileContextPercent(telemetryData); + + if (suffixPercent < 0 || suffixPercent > 100) { + throw new Error(`suffixPercent must be between 0 and 100, but was ${suffixPercent}`); + } + + if (stableContextPercent < 0 || stableContextPercent > 100) { + throw new Error(`stableContextPercent must be between 0 and 100, but was ${stableContextPercent}`); + } + + if (volatileContextPercent < 0 || volatileContextPercent > 100) { + throw new Error(`volatileContextPercent must be between 0 and 100, but was ${volatileContextPercent}`); + } + + const prefixPercent = 100 - suffixPercent - stableContextPercent - volatileContextPercent; + if (prefixPercent <= 1 || prefixPercent > 100) { + throw new Error(`prefixPercent must be between 1 and 100, but was ${prefixPercent}`); + } + + return { + prefix: prefixPercent / 100, + suffix: suffixPercent / 100, + stableContext: stableContextPercent / 100, + volatileContext: volatileContextPercent / 100, + }; + } + + private async createPromptUnsafe( + opts: CompletionsPromptOptions, + cancellationToken?: CancellationToken + ): Promise<PromptResponse> { + this.renderId++; + const { completionId, completionState, telemetryData, promptOpts } = opts; + const failFastPrompt = await this.failFastPrompt(completionState.textDocument, cancellationToken); + if (failFastPrompt) { + return failFastPrompt; + } + + const languageId = completionState.textDocument.detectedLanguageId; + const start = performance.now(); + let contextItems; + if (this.instantiationService.invokeFunction(useContextProviderAPI, languageId, telemetryData)) { + contextItems = await this.resolveContext(completionId, completionState, telemetryData, cancellationToken); + } + const updateDataTimeMs = performance.now() - start; + const renderedComponents: Partial<Record<PromptComponentId, RenderedComponent>> = {}; + const aggregatedMetadata: PromptMetadata = { + renderId: this.renderId, + rendererName: 'w', + tokenizer: promptOpts?.tokenizer ?? TokenizerName.o200k, + elisionTimeMs: 0, + renderTimeMs: 0, + updateDataTimeMs: updateDataTimeMs, + componentStatistics: [], + }; + + const { maxPromptLength } = this.instantiationService.invokeFunction(getPromptOptions, telemetryData, languageId); + const allocation = this.getComponentAllocation(telemetryData); + + const suffixAllocation = allocation.suffix * maxPromptLength; + const estimatedMaxSuffixCost = this.components.suffix.estimatedCost?.(opts, contextItems); + let cascadeOrder: PromptComponentId[] = ['stableContext', 'volatileContext', 'prefix', 'suffix']; + if (suffixAllocation > SMALL_SUFFIX_THRESHOLD * (estimatedMaxSuffixCost ?? 0)) { + cascadeOrder = ['stableContext', 'volatileContext', 'suffix', 'prefix']; + } + + let surplusBudget = 0; + // Allocate excess budget in cascade order + for (const id of cascadeOrder) { + const componentBudget = surplusBudget + maxPromptLength * allocation[id]; + const rendered = renderWithMetadata(this.components[id], componentBudget, opts, contextItems); + surplusBudget = componentBudget - rendered.cost; + renderedComponents[id] = rendered; + aggregateMetadata(aggregatedMetadata, rendered.metadata); + } + + const [prefix, trailingWs] = trimLastLine(renderedComponents.prefix!.text); + + const end = performance.now(); + const contextProvidersTelemetry = this.instantiationService.invokeFunction(useContextProviderAPI, languageId, telemetryData) + ? this.telemetrizeContext( + completionId, + aggregatedMetadata.componentStatistics, + contextItems?.resolvedContextItems ?? [] + ) + : []; + + const context = [ + renderedComponents.stableContext!.text.trim(), + renderedComponents.volatileContext!.text.trim(), + ]; + const prefixWithContext = promptOpts?.separateContext + ? prefix + : // This should not happen, since we always separate context. If it does happen, + // the token counts for the prefix will be wrong, since the workspace context + // will have comment markers. + commentBlockAsSingles(context.join('\n'), languageId) + '\n\n' + prefix; + + return { + type: 'prompt', + prompt: { + prefix: prefixWithContext, + prefixTokens: + renderedComponents.prefix!.cost + + renderedComponents.stableContext!.cost + + renderedComponents.volatileContext!.cost, + suffix: renderedComponents.suffix!.text, + suffixTokens: renderedComponents.suffix!.cost, + context: promptOpts?.separateContext ? context : undefined, + isFimEnabled: renderedComponents.suffix!.text.length > 0, + }, + computeTimeMs: end - start, + trailingWs, + neighborSource: new Map(), + metadata: aggregatedMetadata, + contextProvidersTelemetry, + }; + } + + private async resolveContext( + completionId: string, + completionState: CompletionState, + telemetryData: TelemetryWithExp, + cancellationToken?: CancellationToken + ): Promise<ValidatedContextItems & { resolvedContextItems: ResolvedContextItem[] }> { + const resolvedContextItems: ResolvedContextItem[] = await this.contextProviderBridge.resolution(completionId); + const { textDocument } = completionState; + const matchedContextItems = resolvedContextItems.filter(matchContextItems); + + const traits: TraitWithId[] = this.instantiationService.invokeFunction(getTraitsFromContextItems, completionId, matchedContextItems); + void this.instantiationService.invokeFunction(ReportTraitsTelemetry, + `contextProvider.traits`, + traits, + textDocument.detectedLanguageId, + textDocument.detectedLanguageId, // TextDocumentContext does not have clientLanguageId + telemetryData + ); + + const codeSnippets: CodeSnippetWithId[] = await this.instantiationService.invokeFunction(getCodeSnippetsFromContextItems, + completionId, + matchedContextItems, + textDocument.detectedLanguageId + ); + return { traits, codeSnippets, resolvedContextItems }; + } + + private telemetrizeContext( + completionId: string, + componentStatistics: ComponentStatistics[], + resolvedContextItems: ResolvedContextItem[] + ): ContextProviderTelemetry[] { + const promptMatcher = componentStatisticsToPromptMatcher(componentStatistics); + this.contextProviderStatistics.getStatisticsForCompletion(completionId).computeMatch(promptMatcher); + const contextProvidersTelemetry = telemetrizeContextItems(this.contextProviderStatistics, completionId, resolvedContextItems); + // To support generating context provider metrics of completion in COffE. + logger.debug(this.logTargetService, `Context providers telemetry: '${JSON.stringify(contextProvidersTelemetry)}'`); + return contextProvidersTelemetry; + } + + private async failFastPrompt(textDocument: TextDocumentContents, cancellationToken: CancellationToken | undefined) { + if (cancellationToken?.isCancellationRequested) { + return _promptCancelled; + } + if (await this.ignoreService.isCopilotIgnored(URI.parse(textDocument.uri))) { + return _copilotContentExclusion; + } + + if (textDocument.getText().length < MIN_PROMPT_CHARS) { + // Too short context + return _contextTooShort; + } + } + + private errorPrompt(error: Error): PromptResponse { + telemetryException(this.completionsTelemetryService, error, 'WorkspaceContextPromptFactory'); + return _promptError; + } +} + +function aggregateMetadata(aggregated: PromptMetadata, metadata: PromptMetadata): void { + aggregated.elisionTimeMs += metadata.elisionTimeMs; + aggregated.renderTimeMs += metadata.renderTimeMs; + aggregated.updateDataTimeMs += metadata.updateDataTimeMs; + aggregated.componentStatistics.push(...metadata.componentStatistics); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/completionsPromptFactory.ts b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/completionsPromptFactory.ts new file mode 100644 index 0000000..e0cd59d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/completionsPromptFactory.ts @@ -0,0 +1,132 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken, CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { VirtualPrompt } from '../../../../prompt/src/components/virtualPrompt'; +import { TokenizerName } from '../../../../prompt/src/tokenization'; +import { CompletionState } from '../../completionState'; +import { TelemetryWithExp } from '../../telemetry'; +import { _promptCancelled, _promptError, _promptTimeout, PromptResponse } from '../prompt'; +import { + PromptOrdering, + TestComponentsCompletionsPromptFactory +} from './componentsCompletionsPromptFactory'; +import { createServiceIdentifier } from '../../../../../../../util/common/services'; + +export interface PromptOpts { + data?: unknown; + separateContext?: boolean; + tokenizer?: TokenizerName; +} + +export interface CompletionsPromptOptions { + completionId: string; + completionState: CompletionState; + telemetryData: TelemetryWithExp; + promptOpts?: PromptOpts; +} + +export interface IPromptFactory { + prompt( + opts: CompletionsPromptOptions, + cancellationToken?: CancellationToken + ): Promise<PromptResponse>; +} + +export const ICompletionsPromptFactoryService = createServiceIdentifier<ICompletionsPromptFactoryService>('ICompletionsPromptFactoryService'); +export interface ICompletionsPromptFactoryService extends IPromptFactory { + readonly _serviceBrand: undefined; +} + +// This class needs to extend CompletionsPromptFactory since it's set on the context. +class SequentialCompletionsPromptFactory implements IPromptFactory { + declare _serviceBrand: undefined; + private lastPromise?: Promise<PromptResponse>; + + constructor(private readonly delegate: IPromptFactory) { } + + async prompt(opts: CompletionsPromptOptions, cancellationToken?: CancellationToken): Promise<PromptResponse> { + this.lastPromise = this.promptAsync(opts, cancellationToken); + return this.lastPromise; + } + + private async promptAsync( + opts: CompletionsPromptOptions, + cancellationToken?: CancellationToken + ): Promise<PromptResponse> { + // Wait for previous request to complete + await this.lastPromise; + + // Check if request was cancelled while waiting + if (cancellationToken?.isCancellationRequested) { + return _promptCancelled; + } + + // Return prompt from delegate catching any errors + try { + return await this.delegate.prompt(opts, cancellationToken); + } catch { + return _promptError; + } + } +} + +// 0.01% of prompt construction time is 1s+. Setting this to 1200ms should be safe. +export const DEFAULT_PROMPT_TIMEOUT = 1200; +class TimeoutHandlingCompletionsPromptFactory implements IPromptFactory { + constructor(private readonly delegate: IPromptFactory) { } + + async prompt(opts: CompletionsPromptOptions, cancellationToken?: CancellationToken): Promise<PromptResponse> { + const timeoutTokenSource = new CancellationTokenSource(); + const timeoutToken = timeoutTokenSource.token; + cancellationToken?.onCancellationRequested(() => { + timeoutTokenSource.cancel(); + }); + + return await Promise.race([ + this.delegate.prompt(opts, timeoutToken), + new Promise<PromptResponse>(resolve => { + setTimeout(() => { + // Cancel the token when timeout occurs + timeoutTokenSource.cancel(); + resolve(_promptTimeout); + }, DEFAULT_PROMPT_TIMEOUT); + }), + ]); + } +} + +class BaseComponentsCompletionsPromptFactory implements IPromptFactory { + declare _serviceBrand: undefined; + + private readonly delegate: IPromptFactory; + + constructor( + virtualPrompt: VirtualPrompt | undefined, + ordering: PromptOrdering | undefined, + @IInstantiationService instantiationService: IInstantiationService, + ) { + this.delegate = new SequentialCompletionsPromptFactory( + new TimeoutHandlingCompletionsPromptFactory( + instantiationService.createInstance(TestComponentsCompletionsPromptFactory, virtualPrompt, ordering) + ) + ); + } + + prompt(opts: CompletionsPromptOptions, cancellationToken?: CancellationToken): Promise<PromptResponse> { + return this.delegate.prompt(opts, cancellationToken); + } +} + +export class CompletionsPromptFactory extends BaseComponentsCompletionsPromptFactory { + constructor( + @IInstantiationService instantiationService: IInstantiationService, + ) { + super(undefined, undefined, instantiationService); + } +} + +export class TestCompletionsPromptFactory extends BaseComponentsCompletionsPromptFactory { } \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/componentsCompletionsPromptFactory.tsx b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/componentsCompletionsPromptFactory.tsx new file mode 100644 index 0000000..f029ed8 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/componentsCompletionsPromptFactory.tsx @@ -0,0 +1,514 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ +import { ICompletionsLogTargetService, logger } from '../../logger'; + +import { IIgnoreService } from '../../../../../../../platform/ignore/common/ignoreService'; +import { URI } from '../../../../../../../util/vs/base/common/uri'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTelemetryService } from '../../../../bridge/src/completionsTelemetryServiceBridge'; +import { DataPipe, VirtualPrompt } from '../../../../prompt/src/components/virtualPrompt'; +import { TokenizerName } from '../../../../prompt/src/tokenization'; +import { CancellationToken, Position } from '../../../../types/src'; +import { CompletionState } from '../../completionState'; +import { telemetryException, TelemetryWithExp } from '../../telemetry'; +import { TextDocumentContents } from '../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { CodeSnippets } from '../components/codeSnippets'; +import { CompletionsContext } from '../components/completionsContext'; +import { CompletionsPromptOk, CompletionsPromptRenderer } from '../components/completionsPromptRenderer'; +import { ICompletionsContextProviderBridgeService } from '../components/contextProviderBridge'; +import { CurrentFile } from '../components/currentFile'; +import { DocumentMarker } from '../components/marker'; +import { RecentEdits } from '../components/recentEdits'; +import { SimilarFiles } from '../components/similarFiles'; +import { splitContextCompletionsPrompt } from '../components/splitContextPrompt'; +import { SplitContextPromptRenderer } from '../components/splitContextPromptRenderer'; +import { Traits } from '../components/traits'; + +import { Diagnostics } from '../components/diagnostics'; +import { + ContextProviderTelemetry, + matchContextItems, + ResolvedContextItem, + telemetrizeContextItems, + useContextProviderAPI, +} from '../contextProviderRegistry'; +import { getCodeSnippetsFromContextItems } from '../contextProviders/codeSnippets'; +import { + CodeSnippetWithId, + SupportedContextItemWithId, + TraitWithId, + type DiagnosticBagWithId, +} from '../contextProviders/contextItemSchemas'; +import { getDiagnosticsFromContextItems } from '../contextProviders/diagnostics'; +import { getTraitsFromContextItems, ReportTraitsTelemetry } from '../contextProviders/traits'; +import { componentStatisticsToPromptMatcher, ICompletionsContextProviderService } from '../contextProviderStatistics'; +import { + _contextTooShort, + _copilotContentExclusion, + _promptCancelled, + _promptError, + getPromptOptions, + MIN_PROMPT_CHARS, + PromptResponse, + trimLastLine, +} from '../prompt'; +import { ICompletionsRecentEditsProviderService } from '../recentEdits/recentEditsProvider'; +import { isIncludeNeighborFilesActive } from '../similarFiles/neighborFiles'; +import { + CompletionsPromptOptions, IPromptFactory, + PromptOpts +} from './completionsPromptFactory'; + +export type CompletionRequestDocument = TextDocumentContents; + +export type CompletionRequestData = { + document: CompletionRequestDocument; + position: Position; + telemetryData: TelemetryWithExp; + cancellationToken?: CancellationToken; + // see inlineCompletions data param + data?: unknown; + // Context provider items + traits?: TraitWithId[]; + codeSnippets?: CodeSnippetWithId[]; + diagnostics?: DiagnosticBagWithId[]; + turnOffSimilarFiles?: boolean; + suffixMatchThreshold?: number; + maxPromptTokens: number; + tokenizer?: TokenizerName; +}; + +export function isCompletionRequestData(data: unknown): data is CompletionRequestData { + if (!data || typeof data !== 'object') { return false; } + + const req = data as Partial<CompletionRequestData>; + + // Check document + if (!req.document) { return false; } + + // Check position + if (!req.position) { return false; } + if (req.position.line === undefined) { return false; } + if (req.position.character === undefined) { return false; } + + // Check telemetryData + if (!req.telemetryData) { return false; } + + return true; +} + +export enum PromptOrdering { + Default = 'default', + SplitContext = 'splitContext', +} + +type DeclarativePromptFunction = typeof defaultCompletionsPrompt; +type AvailableDeclarativePrompts = { + [K in PromptOrdering]: { + promptFunction: DeclarativePromptFunction; + renderer: typeof CompletionsPromptRenderer; + }; +}; + +const availableDeclarativePrompts: AvailableDeclarativePrompts = { + [PromptOrdering.Default]: { + promptFunction: defaultCompletionsPrompt, + renderer: CompletionsPromptRenderer, + }, + [PromptOrdering.SplitContext]: { + promptFunction: splitContextCompletionsPrompt, + renderer: SplitContextPromptRenderer, + }, +}; + +// The weights mimic the PromptPriorityList from prompt/src/wishlist.ts +function defaultCompletionsPrompt(accessor: ServicesAccessor) { + const tdms = accessor.get(ICompletionsTextDocumentManagerService); + const instantiationService = accessor.get(IInstantiationService); + const recentEditsProvider = accessor.get(ICompletionsRecentEditsProviderService); + return ( + <> + <CompletionsContext> + <DocumentMarker tdms={tdms} weight={0.7} /> + <Traits weight={0.6} /> + <Diagnostics tdms={tdms} weight={0.65} /> + <CodeSnippets tdms={tdms} weight={0.9} /> + <SimilarFiles tdms={tdms} instantiationService={instantiationService} weight={0.8} /> + <RecentEdits tdms={tdms} recentEditsProvider={recentEditsProvider} weight={0.99} /> + </CompletionsContext> + <CurrentFile weight={1} /> + </> + ); +} + +abstract class BaseComponentsCompletionsPromptFactory implements IPromptFactory { + declare _serviceBrand: undefined; + private virtualPrompt: VirtualPrompt; + private pipe: DataPipe; + private renderer: CompletionsPromptRenderer; + private promptOrdering: PromptOrdering; + + constructor( + virtualPrompt: VirtualPrompt | undefined, + ordering: PromptOrdering | undefined, + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsTelemetryService private readonly completionsTelemetryService: ICompletionsTelemetryService, + @IIgnoreService private readonly ignoreService: IIgnoreService, + @ICompletionsContextProviderBridgeService private readonly contextProviderBridge: ICompletionsContextProviderBridgeService, + @ICompletionsLogTargetService private readonly logTarget: ICompletionsLogTargetService, + @ICompletionsContextProviderService private readonly contextProviderStatistics: ICompletionsContextProviderService, + ) { + this.promptOrdering = ordering ?? PromptOrdering.Default; + this.virtualPrompt = virtualPrompt ?? new VirtualPrompt(this.completionsPrompt()); + this.pipe = this.virtualPrompt.createPipe(); + this.renderer = this.getRenderer(); + } + + async prompt(opts: CompletionsPromptOptions, cancellationToken?: CancellationToken): Promise<PromptResponse> { + try { + return await this.createPromptUnsafe(opts, cancellationToken); + } catch (e) { + return this.errorPrompt(e as Error); + } + } + + async createPromptUnsafe( + { completionId, completionState, telemetryData, promptOpts }: CompletionsPromptOptions, + cancellationToken?: CancellationToken + ): Promise<PromptResponse> { + const { maxPromptLength, suffixPercent, suffixMatchThreshold } = this.instantiationService.invokeFunction(getPromptOptions, + telemetryData, + completionState.textDocument.detectedLanguageId + ); + + const failFastPrompt = await this.failFastPrompt( + completionState.textDocument, + completionState.position, + suffixPercent, + cancellationToken + ); + if (failFastPrompt) { + return failFastPrompt; + } + + // TODO: Prompt ordering changes are triggered by ExP changes. + // TODO@benibenj remove this as its always true (except in tests) + const promptOrdering = promptOpts?.separateContext ? PromptOrdering.SplitContext : PromptOrdering.Default; + this.setPromptOrdering(promptOrdering); + + const start = performance.now(); + + const { traits, codeSnippets, diagnostics, turnOffSimilarFiles, resolvedContextItems } = await this.resolveContext( + completionId, + completionState, + telemetryData, + cancellationToken, + promptOpts + ); + + await this.updateComponentData( + completionState.textDocument, + completionState.position, + traits, + codeSnippets, + diagnostics, + telemetryData, + turnOffSimilarFiles, + maxPromptLength, + cancellationToken, + promptOpts, + suffixMatchThreshold, + promptOpts?.tokenizer + ); + + if (cancellationToken?.isCancellationRequested) { + return _promptCancelled; + } + + const snapshot = this.virtualPrompt.snapshot(cancellationToken); + const snapshotStatus = snapshot.status; + if (snapshotStatus === 'cancelled') { + return _promptCancelled; + } else if (snapshotStatus === 'error') { + return this.errorPrompt(snapshot.error); + } + + const rendered = this.renderer.render( + snapshot.snapshot!, + { + delimiter: '\n', + tokenizer: promptOpts?.tokenizer, + promptTokenLimit: maxPromptLength, + suffixPercent: suffixPercent, + languageId: completionState.textDocument.detectedLanguageId, + }, + cancellationToken + ); + if (rendered.status === 'cancelled') { + return _promptCancelled; + } else if (rendered.status === 'error') { + return this.errorPrompt(rendered.error); + } + + const [prefix, trailingWs] = trimLastLine(rendered.prefix); + const renderedTrimmed = { ...rendered, prefix }; + + let contextProvidersTelemetry: ContextProviderTelemetry[] | undefined = undefined; + const languageId = completionState.textDocument.detectedLanguageId; + if (this.instantiationService.invokeFunction(useContextProviderAPI, languageId, telemetryData)) { + const promptMatcher = componentStatisticsToPromptMatcher(rendered.metadata.componentStatistics); + this.contextProviderStatistics + .getStatisticsForCompletion(completionId) + .computeMatch(promptMatcher); + contextProvidersTelemetry = telemetrizeContextItems(this.contextProviderStatistics, completionId, resolvedContextItems); + // To support generating context provider metrics of completion in COffE. + logger.debug(this.logTarget, `Context providers telemetry: '${JSON.stringify(contextProvidersTelemetry)}'`); + } + const end = performance.now(); + this.resetIfEmpty(rendered); + return this.successPrompt(renderedTrimmed, end, start, trailingWs, contextProvidersTelemetry); + } + + private async updateComponentData( + textDocument: CompletionRequestDocument, + position: Position, + traits: TraitWithId[] | undefined, + codeSnippets: CodeSnippetWithId[] | undefined, + diagnostics: DiagnosticBagWithId[] | undefined, + telemetryData: TelemetryWithExp, + turnOffSimilarFiles: boolean, + maxPromptLength: number, + cancellationToken?: CancellationToken, + opts: PromptOpts = {}, + suffixMatchThreshold?: number, + tokenizer?: TokenizerName + ) { + const completionRequestData = this.createRequestData( + textDocument, + position, + telemetryData, + cancellationToken, + opts, + maxPromptLength, + traits, + codeSnippets, + diagnostics, + turnOffSimilarFiles, + suffixMatchThreshold, + tokenizer + ); + await this.pipe.pump(completionRequestData); + } + + private async resolveContext( + completionId: string, + completionState: CompletionState, + telemetryData: TelemetryWithExp, + cancellationToken?: CancellationToken, + opts: PromptOpts = {} + ): Promise<{ + traits: TraitWithId[] | undefined; + codeSnippets: CodeSnippetWithId[] | undefined; + diagnostics: DiagnosticBagWithId[] | undefined; + turnOffSimilarFiles: boolean; + resolvedContextItems: ResolvedContextItem[]; + }> { + let resolvedContextItems: ResolvedContextItem[] = []; + let traits: TraitWithId[] | undefined; + let codeSnippets: CodeSnippetWithId[] | undefined; + let diagnostics: DiagnosticBagWithId[] | undefined; + let turnOffSimilarFiles = false; + if (this.instantiationService.invokeFunction(useContextProviderAPI, completionState.textDocument.detectedLanguageId, telemetryData)) { + resolvedContextItems = await this.contextProviderBridge.resolution(completionId); + const { textDocument } = completionState; + // Turn off neighboring files if: + // - it's not explicitly enabled via EXP flag + // - there are matched context providers + const matchedContextItems = resolvedContextItems.filter(matchContextItems); + if (!this.instantiationService.invokeFunction(similarFilesEnabled, textDocument.detectedLanguageId, matchedContextItems, telemetryData)) { + turnOffSimilarFiles = true; + } + + traits = await this.instantiationService.invokeFunction(getTraitsFromContextItems, completionId, matchedContextItems); + void this.instantiationService.invokeFunction(ReportTraitsTelemetry, + `contextProvider.traits`, + traits, + textDocument.detectedLanguageId, + textDocument.detectedLanguageId, // TextDocumentContext does not have clientLanguageId + telemetryData + ); + + codeSnippets = await this.instantiationService.invokeFunction(getCodeSnippetsFromContextItems, + completionId, + matchedContextItems, + textDocument.detectedLanguageId + ); + + diagnostics = await this.instantiationService.invokeFunction(getDiagnosticsFromContextItems, + completionId, + matchedContextItems + ); + } + return { traits, codeSnippets, diagnostics, turnOffSimilarFiles, resolvedContextItems }; + } + + private async failFastPrompt( + textDocument: TextDocumentContents, + position: Position, + suffixPercent: number, + cancellationToken: CancellationToken | undefined + ) { + if (cancellationToken?.isCancellationRequested) { + return _promptCancelled; + } + if (await this.ignoreService.isCopilotIgnored(URI.parse(textDocument.uri))) { + return _copilotContentExclusion; + } + + const eligibleChars = suffixPercent > 0 ? textDocument.getText().length : textDocument.offsetAt(position); + if (eligibleChars < MIN_PROMPT_CHARS) { + // Too short context + return _contextTooShort; + } + } + + private createRequestData( + textDocument: CompletionRequestDocument, + position: Position, + telemetryData: TelemetryWithExp, + cancellationToken: CancellationToken | undefined, + opts: PromptOpts, + maxPromptLength: number, + traits?: TraitWithId[], + codeSnippets?: CodeSnippetWithId[], + diagnostics?: DiagnosticBagWithId[], + turnOffSimilarFiles?: boolean, + suffixMatchThreshold?: number, + tokenizer?: TokenizerName + ): CompletionRequestData { + return { + document: textDocument, + position, + telemetryData, + cancellationToken, + data: opts.data, + traits, + codeSnippets, + diagnostics, + turnOffSimilarFiles, + suffixMatchThreshold, + maxPromptTokens: maxPromptLength, + tokenizer, + }; + } + + private resetIfEmpty(rendered: CompletionsPromptOk) { + if (rendered.prefix.length === 0 && rendered.suffix.length === 0) { + this.reset(); + } + } + + private successPrompt( + rendered: CompletionsPromptOk, + end: number, + start: number, + trailingWs: string, + contextProvidersTelemetry?: ContextProviderTelemetry[] + ): PromptResponse { + return { + type: 'prompt', + prompt: { + prefix: rendered.prefix, + prefixTokens: rendered.prefixTokens, + suffix: rendered.suffix, + suffixTokens: rendered.suffixTokens, + context: rendered.context, + isFimEnabled: rendered.suffix.length > 0, + }, + computeTimeMs: end - start, + trailingWs, + neighborSource: new Map(), + metadata: rendered.metadata, + contextProvidersTelemetry, + }; + } + + private errorPrompt(error: Error): PromptResponse { + telemetryException(this.completionsTelemetryService, error, 'PromptComponents.CompletionsPromptFactory'); + this.reset(); + return _promptError; + } + + private reset() { + this.renderer = this.getRenderer(); + this.virtualPrompt = new VirtualPrompt(this.completionsPrompt()); + this.pipe = this.virtualPrompt.createPipe(); + } + + private setPromptOrdering(ordering: PromptOrdering) { + if (this.promptOrdering !== ordering) { + this.promptOrdering = ordering; + this.reset(); + } + } + + private completionsPrompt() { + const promptFunction = + availableDeclarativePrompts[this.promptOrdering]?.promptFunction ?? defaultCompletionsPrompt; + return this.instantiationService.invokeFunction(promptFunction); + } + + private getRenderer() { + const promptInfo = + availableDeclarativePrompts[this.promptOrdering] ?? availableDeclarativePrompts[PromptOrdering.Default]; + return new promptInfo.renderer(); + } +} + +export class ComponentsCompletionsPromptFactory extends BaseComponentsCompletionsPromptFactory { + constructor( + @IInstantiationService instantiationService: IInstantiationService, + @ICompletionsTelemetryService completionsTelemetryService: ICompletionsTelemetryService, + @IIgnoreService ignoreService: IIgnoreService, + @ICompletionsContextProviderBridgeService contextProviderBridge: ICompletionsContextProviderBridgeService, + @ICompletionsLogTargetService logTarget: ICompletionsLogTargetService, + @ICompletionsContextProviderService contextProviderStatistics: ICompletionsContextProviderService, + ) { + super( + undefined, + undefined, + instantiationService, + completionsTelemetryService, + ignoreService, + contextProviderBridge, + logTarget, + contextProviderStatistics + ); + } +} + +export class TestComponentsCompletionsPromptFactory extends BaseComponentsCompletionsPromptFactory { } + +// Similar files is enabled if: +// - the languageId is C/C++. +// - it's explicitly enabled via EXP flag or config. +// - no code snippets are provided (which includes the case when all providers error). +function similarFilesEnabled( + accessor: ServicesAccessor, + detectedLanguageId: string, + matchedContextItems: ResolvedContextItem<SupportedContextItemWithId>[], + telemetryData: TelemetryWithExp +) { + const cppLanguageIds = ['cpp', 'c']; + const includeNeighboringFiles = + isIncludeNeighborFilesActive(accessor, detectedLanguageId, telemetryData) || cppLanguageIds.includes(detectedLanguageId); + return ( + includeNeighboringFiles || !matchedContextItems.some(ci => ci.data.some(item => item.type === 'CodeSnippet')) + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/test/completionsPromptFactory.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/test/completionsPromptFactory.test.tsx new file mode 100644 index 0000000..58efa9d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/completionsPromptFactory/test/completionsPromptFactory.test.tsx @@ -0,0 +1,935 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import dedent from 'ts-dedent'; +import { Diagnostic, DiagnosticSeverity, Range, Uri } from 'vscode'; +import { CancellationTokenSource, Position } from 'vscode-languageserver-protocol'; +import { MutableObservableWorkspace } from '../../../../../../../../platform/inlineEdits/common/observableWorkspace'; +import { TestingServiceCollection } from '../../../../../../../../platform/test/node/services'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ComponentContext, PromptElementProps, Text } from '../../../../../prompt/src/components/components'; +import { Dispatch, StateUpdater } from '../../../../../prompt/src/components/hooks'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { DEFAULT_MAX_COMPLETION_LENGTH } from '../../../../../prompt/src/prompt'; +import { getTokenizer, TokenizerName } from '../../../../../prompt/src/tokenization'; +import { CodeSnippet, ContextProvider, SupportedContextItem, Trait, type DiagnosticBag } from '../../../../../types/src'; +import { ICompletionsObservableWorkspace } from '../../../completionsObservableWorkspace'; +import { createCompletionState } from '../../../completionState'; +import { ConfigKey, ICompletionsConfigProvider, InMemoryConfigProvider } from '../../../config'; +import { ICompletionsFeaturesService } from '../../../experiments/featuresService'; +import { TelemetryWithExp } from '../../../telemetry'; +import { createLibTestingContext } from '../../../test/context'; +import { withInMemoryTelemetry } from '../../../test/telemetry'; +import { createTextDocument, TestTextDocumentManager } from '../../../test/textDocument'; +import { ITextDocument } from '../../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; +import { CompletionsContext } from '../../components/completionsContext'; +import { ICompletionsContextProviderBridgeService } from '../../components/contextProviderBridge'; +import { CurrentFile } from '../../components/currentFile'; +import { ContextProviderTelemetry, ICompletionsContextProviderRegistryService } from '../../contextProviderRegistry'; +import { _contextTooShort, _promptCancelled, _promptError } from '../../prompt'; +import { FullRecentEditsProvider, ICompletionsRecentEditsProviderService } from '../../recentEdits/recentEditsProvider'; +import { NeighborSource } from '../../similarFiles/neighborFiles'; +import { + DEFAULT_PROMPT_TIMEOUT, IPromptFactory, + TestCompletionsPromptFactory +} from '../completionsPromptFactory'; +import { + isCompletionRequestData, + PromptOrdering, + TestComponentsCompletionsPromptFactory +} from '../componentsCompletionsPromptFactory'; + +suite('Completions Prompt Factory', function () { + let telemetryData: TelemetryWithExp; + let accessor: ServicesAccessor; + let serviceCollection: TestingServiceCollection; + let clock: sinon.SinonFakeTimers | undefined; + let cts: CancellationTokenSource; + const longPrefix = Array.from({ length: 60 }, (_, i) => `const a${i} = ${i};`).join('\n'); + const defaultTextDocument = createTextDocument( + 'file:///path/basename', + 'typescript', + 0, + dedent` + ${longPrefix} + function f| + const b = 2; + ` + ); + let promptFactory: IPromptFactory; + + function invokePromptFactory( + opts: { + completionId?: string; + textDocument?: ITextDocument; + position?: Position; + separateContext?: boolean; + } = {}, + factory: IPromptFactory = promptFactory, + ) { + const textDocument = opts.textDocument ?? defaultTextDocument; + const position = opts.position ?? textDocument.positionAt(textDocument.getText().indexOf('|')); + const completionState = createCompletionState(textDocument, position); + const separateContext = opts.separateContext ?? false; + const completionId = opts.completionId ?? 'completion_id'; + const contextProviderBridge = accessor.get(ICompletionsContextProviderBridgeService); + contextProviderBridge.schedule(completionState, completionId, 'opId', telemetryData); + return factory.prompt( + { completionId, completionState, telemetryData, promptOpts: { separateContext } }, + cts.token + ); + } + + setup(function () { + serviceCollection = createLibTestingContext(); + accessor = serviceCollection.createTestingAccessor(); + telemetryData = TelemetryWithExp.createEmptyConfigForTesting(); + cts = new CancellationTokenSource(); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, undefined, undefined); + }); + + teardown(function () { + clock?.restore(); + sinon.restore(); + NeighborSource.reset(); + }); + + test('prompt should include document marker', async function () { + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.prefix, `// Path: basename\n${longPrefix}\nfunction f`); + assert.deepStrictEqual(result.prompt.prefixTokens, 427); + assert.deepStrictEqual(result.prompt.suffix, 'const b = 2;'); + assert.deepStrictEqual(result.prompt.suffixTokens, 6); + }); + + test('prompt should include neighboring files', async function () { + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument('file:///something.ts', 'typescript', '// match function f\nfunction foo() {}'); + + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual( + result.prompt.prefix, + dedent` + // Path: basename + // Compare this snippet from something.ts: + // // match function f + // function foo() {} + ${longPrefix} + function f + ` + ); + assert.deepStrictEqual(result.prompt.prefixTokens, 446); + assert.deepStrictEqual(result.prompt.suffix, 'const b = 2;'); + assert.deepStrictEqual(result.prompt.suffixTokens, 6); + }); + + test('prompt should include recent edits', async function () { + const serviceCollectionClone = serviceCollection.clone(); + const workspace = new CompletionsMutableObservableWorkspace(); + serviceCollectionClone.define(ICompletionsObservableWorkspace, workspace); + + // TODO: figure out how to simulate real document update events + const rep = new MockRecentEditsProvider(undefined, workspace); + serviceCollectionClone.define(ICompletionsRecentEditsProviderService, rep); + + const accessorClone = serviceCollectionClone.createTestingAccessor(); + const promptFactory = accessorClone.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, undefined, undefined); + + // Ensure the document is open + const tdm = accessorClone.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument(defaultTextDocument.uri, defaultTextDocument.languageId, defaultTextDocument.getText()); + + // Update the distance setting to avoid having to create a huge document + rep.config.activeDocDistanceLimitFromCursor = 10; + + rep.testUpdateRecentEdits(defaultTextDocument.uri, defaultTextDocument.getText()); + rep.testUpdateRecentEdits( + defaultTextDocument.uri, + defaultTextDocument.getText().replace('const a0', 'const c1') + ); + + const result = await invokePromptFactory({}, promptFactory); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual( + result.prompt.prefix, + dedent` + // Path: basename + // These are recently edited files. Do not suggest code that has been deleted. + // File: basename + // --- a/file:///path/basename + // +++ b/file:///path/basename + // @@ -1,4 +1,4 @@ + // +const c1 = 0; + // -const a0 = 0; --- IGNORE --- + // const a1 = 1; + // const a2 = 2; + // const a3 = 3; + // End of recent edits + ${longPrefix} + function f + ` + ); + assert.deepStrictEqual(result.prompt.suffix, 'const b = 2;'); + }); + + test('recent edits are removed as a chunk', async function () { + const serviceCollectionClone = serviceCollection.clone(); + const workspace = new CompletionsMutableObservableWorkspace(); + serviceCollectionClone.define(ICompletionsObservableWorkspace, workspace); + // TODO: figure out how to simulate real document update events + const rep = new MockRecentEditsProvider(undefined, workspace); + serviceCollectionClone.define(ICompletionsRecentEditsProviderService, rep); + + const accessorClone = serviceCollectionClone.createTestingAccessor(); + const promptFactory = accessorClone.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, undefined, undefined); + const featuresService = accessorClone.get(ICompletionsFeaturesService); + + // Ensure the document is open + const tdm = accessorClone.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument(defaultTextDocument.uri, defaultTextDocument.languageId, defaultTextDocument.getText()); + + // Update the distance setting to avoid having to create a huge document + rep.config.activeDocDistanceLimitFromCursor = 10; + + rep.testUpdateRecentEdits(defaultTextDocument.uri, defaultTextDocument.getText()); + rep.testUpdateRecentEdits( + defaultTextDocument.uri, + defaultTextDocument.getText().replace('const a0', 'const c1') + ); + + featuresService.maxPromptCompletionTokens = () => 530 + DEFAULT_MAX_COMPLETION_LENGTH; + featuresService.suffixPercent = () => 0; + + const result = await invokePromptFactory({}, promptFactory); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual( + result.prompt.prefix, + dedent` + // Path: basename + ${longPrefix} + function f + ` + ); + }); + + test('prompt should include context and prefix', async function () { + const result = await invokePromptFactory({ separateContext: true }); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.prefix, `${longPrefix}\nfunction f`); + assert.deepStrictEqual(result.prompt.context, ['Path: basename']); + assert.deepStrictEqual(result.prompt.suffix, 'const b = 2;'); + }); + + test('prompt should include prefix and suffix tokens', async function () { + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.prefixTokens, 427); + assert.deepStrictEqual(result.prompt.suffixTokens, 6); + }); + + test('suffix should be cached if similar enough', async function () { + telemetryData.filtersAndExp.exp.variables.copilotsuffixmatchthreshold = 20; + // Call it once to cache + await invokePromptFactory(); + + const textDocument = createTextDocument( + 'untitled:', + 'typescript', + 1, + dedent` + const a = 1; + function f| + const b = 1; + ` + ); + + const result = await invokePromptFactory({ textDocument }); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.suffix, 'const b = 2;'); + }); + + test('produces timeout prompt if timeout is exceeded', async function () { + clock = sinon.useFakeTimers(); + const TimeoutComponent = (_: PromptElementProps, context: ComponentContext) => { + context.useData(isCompletionRequestData, async _ => { + await clock?.tickAsync(DEFAULT_PROMPT_TIMEOUT + 1); + }); + return <Text>A really cool prompt</Text>; + }; + const virtualPrompt = new VirtualPrompt( + ( + <> + <CompletionsContext> + <TimeoutComponent /> + </CompletionsContext> + <CurrentFile /> + </> + ) + ); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result.type, 'promptTimeout'); + }); + + test('produces valid prompts with multiple promises racing', async function () { + const promises = []; + for (let i = 0; i < 3; i++) { + const textDocument = createTextDocument(`file:///path/basename${i}`, 'typescript', 0, `const a = ${i}|;`); + const promise = invokePromptFactory({ textDocument }); + promises.push(promise); + } + + const results = await Promise.all(promises); + + for (let i = 0; i < 3; i++) { + const result = results[i]; + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.prefix, `// Path: basename${i}\nconst a = ${i}`); + } + }); + + test('handles errors with multiple promises racing', async function () { + sinon + .stub(TestComponentsCompletionsPromptFactory.prototype, 'createPromptUnsafe') + .callThrough() + .onFirstCall() + .throws(new Error('Intentional error')); + + const doc = createTextDocument('file:///path/basename', 'typescript', 0, `const a = 1|;`); + + const smallDoc = createTextDocument('file:///path/basename', 'typescript', 0, `c|`); + + const errorPromise = invokePromptFactory({ textDocument: doc }); + const goodPromise = invokePromptFactory({ textDocument: doc }); + const shortContextPromise = invokePromptFactory({ textDocument: smallDoc }); + + const results = await Promise.all([errorPromise, goodPromise, shortContextPromise]); + + assert.deepStrictEqual(results[0], _promptError); + assert.deepStrictEqual(results[2], _contextTooShort); + + const firstResult = results[1]; + assert.deepStrictEqual(firstResult.type, 'prompt'); + assert.deepStrictEqual(firstResult.prompt.prefix, `// Path: basename\nconst a = 1`); + }); + + test('produces valid prompts with sequential context provider calls', async function () { + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.contextProviders = () => ['traitsProvider']; + + let id = 0; + const traitsProvider: ContextProvider<Trait> = { + id: 'traitsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => { + const traitId = id++; + return Promise.resolve([ + { name: `test_trait${traitId}`, value: 'test_value', id: `trait${traitId}` }, + ]); + }, + }, + }; + accessor.get(ICompletionsContextProviderRegistryService).registerContextProvider(traitsProvider); + + const promises = []; + for (let i = 0; i < 3; i++) { + const textDocument = createTextDocument(`file:///path/basename${i}`, 'typescript', 0, `const a = ${i}|;`); + const promise = invokePromptFactory({ textDocument, completionId: `completion_id_${i}` }); + promises.push(promise); + } + + const results = await Promise.all(promises); + + for (let i = 0; i < 3; i++) { + const result = results[i]; + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual( + result.prompt.prefix, + `// Path: basename${i}\n// Consider this related information:\n// test_trait${i}: test_value\nconst a = ${i}` + ); + assert.deepStrictEqual(result.contextProvidersTelemetry?.length, 1); + assert.deepStrictEqual(result.contextProvidersTelemetry?.[0].usageDetails?.length, 1); + assert.deepStrictEqual(result.contextProvidersTelemetry?.[0].usageDetails?.[0].id, `trait${i}`); + } + }); + + test('produces valid prompts with multiple promises racing, one blocking', async function () { + clock = sinon.useFakeTimers(); + let timeoutMs = DEFAULT_PROMPT_TIMEOUT + 1; + const TimeoutComponent = (_: PromptElementProps, context: ComponentContext) => { + context.useData(isCompletionRequestData, async _ => { + const timeoutPromise = clock?.tickAsync(timeoutMs); + timeoutMs = 0; + await timeoutPromise; + }); + + return <Text>A really cool prompt</Text>; + }; + const virtualPrompt = new VirtualPrompt( + ( + <> + <CompletionsContext> + <TimeoutComponent /> + </CompletionsContext> + <CurrentFile /> + </> + ) + ); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + + const promises = []; + for (let i = 0; i < 2; i++) { + const textDocument = createTextDocument(`file:///${i}`, 'typescript', 0, `const a = ${i}|;`); + const promise = invokePromptFactory({ textDocument }); + promises.push(promise); + } + + const results = await Promise.all(promises); + + assert.deepStrictEqual(results[0].type, 'promptTimeout'); + assert.deepStrictEqual(results[1].type, 'prompt'); + assert.deepStrictEqual(results[1].prompt.prefix, '// A really cool prompt\nconst a = 1'); + }); + + test('token limits can be controlled via EXP', async function () { + const tokenizer = getTokenizer(); + const longText = Array.from({ length: 1000 }, (_, i) => `const a${i} = ${i};`).join('\n'); + const longTextDocument = createTextDocument( + 'file:///path/basename', + 'typescript', + 0, + longText + 'function f|\nconst b = 2;' + ); + const defaultLimitsPrompt = await invokePromptFactory({ textDocument: longTextDocument }); + + assert.deepStrictEqual(defaultLimitsPrompt.type, 'prompt'); + assert.deepStrictEqual(tokenizer.tokenLength(defaultLimitsPrompt.prompt.prefix), 7007); + assert.deepStrictEqual(tokenizer.tokenLength(defaultLimitsPrompt.prompt.suffix), 6); + + // 100 tokens are left for the prompt, 5 are used for the suffix token, so 95 are left + telemetryData.filtersAndExp.exp.variables.maxpromptcompletionTokens = + 100 + // Prefix + suffix + 5 + // Suffix encoding + DEFAULT_MAX_COMPLETION_LENGTH; + telemetryData.filtersAndExp.exp.variables.CopilotSuffixPercent = 2; + + const expLimitsPrompt = await invokePromptFactory({ textDocument: longTextDocument }); + + assert.deepStrictEqual(expLimitsPrompt.type, 'prompt'); + assert.deepStrictEqual(tokenizer.tokenLength(expLimitsPrompt.prompt.prefix), 98); + assert.deepStrictEqual(tokenizer.tokenLength(expLimitsPrompt.prompt.suffix), 2); + }); + + test('produces context too short', async function () { + const tinyTextDocument = createTextDocument('file:///path/basename', 'typescript', 0, ''); + const result = await invokePromptFactory({ textDocument: tinyTextDocument }); + + assert.deepStrictEqual(result, _contextTooShort); + }); + + test('errors when hitting fault barrier', async function () { + const virtualPrompt = new VirtualPrompt(<></>); + virtualPrompt.snapshot = sinon.stub().throws(new Error('Intentional snapshot error')); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result, _promptError); + }); + + test('recovers from error when hitting fault barrier', async function () { + const virtualPrompt = new VirtualPrompt(<></>); + virtualPrompt.snapshot = sinon.stub().throws(new Error('Intentional snapshot error')); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + + let result = await invokePromptFactory(); + assert.deepStrictEqual(result, _promptError); + + result = await invokePromptFactory(); + assert.deepStrictEqual(result.type, 'prompt'); + }); + + test('errors on snapshot error', async function () { + const virtualPrompt = new VirtualPrompt(<></>); + virtualPrompt.snapshot = sinon + .stub() + .returns({ snapshot: undefined, status: 'error', error: new Error('Intentional snapshot error') }); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result, _promptError); + }); + + test('recovers from error on snapshot error', async function () { + const virtualPrompt = new VirtualPrompt(<></>); + virtualPrompt.snapshot = sinon + .stub() + .returns({ snapshot: undefined, status: 'error', error: new Error('Intentional snapshot error') }); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + + let result = await invokePromptFactory(); + assert.deepStrictEqual(result, _promptError); + + result = await invokePromptFactory(); + assert.deepStrictEqual(result.type, 'prompt'); + }); + + test('handles cancellation', async function () { + cts.cancel(); + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result, _promptCancelled); + }); + + test('handles cancellation during update data', async function () { + const CancellationComponent = (_: PromptElementProps, context: ComponentContext) => { + context.useData(isCompletionRequestData, _ => { + cts.cancel(); + }); + return <Text>A really cool prompt</Text>; + }; + const virtualPrompt = new VirtualPrompt( + ( + <> + <CancellationComponent /> + <CurrentFile /> + </> + ) + ); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result, _promptCancelled); + }); + + test('error in snapshot leads to prompt error', async function () { + let outerSetShouldThrowError: Dispatch<StateUpdater<boolean>> = () => { }; + const ErrorThrowingComponent = (_props: PromptElementProps, context: ComponentContext) => { + const [shouldThrowError, setShouldThrowError] = context.useState(false); + outerSetShouldThrowError = setShouldThrowError; + + if (shouldThrowError) { + throw new Error('Intentional error'); + } + return <></>; + }; + const virtualPrompt = new VirtualPrompt(<ErrorThrowingComponent />); + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, undefined); + + outerSetShouldThrowError(true); + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result, _promptError); + }); + + test('prompt should not include context provider info if the context provider API is not enabled', async function () { + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.ContextProviders, []); + + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = ''; + + const result = await invokePromptFactory(); + assert.deepStrictEqual(result.type, 'prompt'); + assert.ok(result.prompt.prefix.includes('Consider this related information:') === false); + }); + + test('prompt should include traits, diagnostics and code snippets if the context provider API is enabled', async function () { + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = 'traitsProvider,diagnosticsProvider,codeSnippetsProvider'; + + const traitsProvider: ContextProvider<Trait> = { + id: 'traitsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => Promise.resolve([{ name: 'test_trait', value: 'test_value' }]), + }, + }; + const diagnosticsProvider: ContextProvider<DiagnosticBag> = { + id: 'diagnosticsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => { + const diag1 = new Diagnostic(new Range(0, 10, 0, 20), 'type exists', DiagnosticSeverity.Error); + diag1.code = 1017; + diag1.source = 'ts'; + const diag2 = new Diagnostic(new Range(0, 20, 0, 25), 'unknown type', DiagnosticSeverity.Warning); + diag2.code = 2017; + return Promise.resolve([{ uri: Uri.file('something.ts'), values: [diag1, diag2] }]); + }, + }, + }; + const codeSnippetsProvider: ContextProvider<CodeSnippet> = { + id: 'codeSnippetsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => Promise.resolve([{ uri: 'file:///something.ts', value: 'function foo() { return 1; }' }]), + }, + }; + const contextProviderRegistry = accessor.get(ICompletionsContextProviderRegistryService); + contextProviderRegistry.registerContextProvider(traitsProvider); + contextProviderRegistry.registerContextProvider(diagnosticsProvider); + contextProviderRegistry.registerContextProvider(codeSnippetsProvider); + + // Register the documents for content exclusion + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument('file:///something.ts', 'typescript', 'does not matter'); + + const result = await invokePromptFactory(); + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual( + result.prompt.prefix, + dedent` + // Path: basename + // Consider this related information: + // test_trait: test_value + // Consider the following typescript diagnostics from something.ts: + // 1:11 - error TS1017: type exists + // 1:21 - warning 2017: unknown type + // Compare this snippet from something.ts: + // function foo() { return 1; } + ` + `\n${longPrefix}\nfunction f` + ); + }); + + test('should still produce a prompt if a context provider errors', async function () { + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = 'errorProvider,codeSnippetsProvider'; + + const errorProvider: ContextProvider<SupportedContextItem> = { + id: 'errorProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: (): Promise<never> => Promise.reject(new Error('Intentional error')), + }, + }; + const codeSnippetsProvider: ContextProvider<CodeSnippet> = { + id: 'codeSnippetsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => Promise.resolve([{ uri: 'file:///something.ts', value: 'function foo() { return 1; }' }]), + }, + }; + const contextProviderRegistry = accessor.get(ICompletionsContextProviderRegistryService); + contextProviderRegistry.registerContextProvider(errorProvider); + contextProviderRegistry.registerContextProvider(codeSnippetsProvider); + + // Register the documents for content exclusion + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument('file:///something.ts', 'typescript', 'does not matter'); + + const result = await invokePromptFactory(); + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual( + result.prompt.prefix, + dedent` + // Path: basename + // Compare this snippet from something.ts: + // function foo() { return 1; } + ` + `\n${longPrefix}\nfunction f` + ); + }); + + test('prompt should include compute time', async function () { + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.ok(result.computeTimeMs > 0); + }); + + test('prompt should trim prefix and include trailingWs', async function () { + const textDocument = createTextDocument( + 'file:///path/basename', + 'typescript', + 0, + `const a = 1;\nfunction f\n const b = 2;\n ` + ); + const result = await invokePromptFactory({ textDocument, position: Position.create(3, 4) }); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.prefix, '// Path: basename\nconst a = 1;\nfunction f\n const b = 2;\n'); + assert.deepStrictEqual(result.trailingWs, ' '); + }); + + test('prompt respects context blocks if separateContext is true', async function () { + function splitContextPrompt() { + return ( + <> + <CompletionsContext> + <Text>First context block</Text> + </CompletionsContext> + <CompletionsContext> + <Text>Second context block</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + } + + const virtualPrompt = new VirtualPrompt(splitContextPrompt()); + + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, PromptOrdering.SplitContext); + const result = await invokePromptFactory({ separateContext: true }); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.context, ['First context block', 'Second context block']); + }); + + test('prompt does not output separate context blocks if separateContext is not specified', async function () { + function splitContextPrompt() { + return ( + <> + <CompletionsContext> + <Text>First context block</Text> + </CompletionsContext> + <CompletionsContext> + <Text>Second context block</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + } + + const virtualPrompt = new VirtualPrompt(splitContextPrompt()); + + promptFactory = accessor.get(IInstantiationService).createInstance(TestCompletionsPromptFactory, virtualPrompt, PromptOrdering.SplitContext); + const result = await invokePromptFactory(); + + assert.deepStrictEqual(result.type, 'prompt'); + assert.deepStrictEqual(result.prompt.context, undefined); + }); + + test('produces metadata', async function () { + const result = await invokePromptFactory(); + assert.deepStrictEqual(result.type, 'prompt'); + + const metadata = result.metadata; + assert.ok(metadata); + assert.ok(metadata.renderId === 0); + assert.ok(metadata.elisionTimeMs > 0); + assert.ok(metadata.renderTimeMs > 0); + assert.ok(metadata.updateDataTimeMs > 0); + assert.deepStrictEqual(metadata.rendererName, 'c'); + assert.deepStrictEqual(metadata.tokenizer, TokenizerName.o200k); + + const componentsUpdateDataTimeMs = metadata.componentStatistics.reduce( + (acc, { updateDataTimeMs }) => acc + (updateDataTimeMs ?? 0), + 0 + ); + assert.ok(componentsUpdateDataTimeMs > 0); + const actualStatsFiltered = metadata.componentStatistics.map(stats => { + if (stats.updateDataTimeMs) { + stats.updateDataTimeMs = 42; + } + return stats; + }); + + assert.deepStrictEqual(actualStatsFiltered, [ + { + componentPath: '$.f[0].CompletionsContext[0].DocumentMarker', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[1].Traits', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[2].Diagnostics', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[3].CodeSnippets', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[4].SimilarFiles', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[5].RecentEdits', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[1].CurrentFile', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[0].DocumentMarker[0].PathMarker[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 422, + actualTokens: 422, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 6, + actualTokens: 6, + }, + ]); + }); + + test('telemetry should include context providers', async function () { + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = 'traitsProvider,codeSnippetsProvider'; + + const traitsContextProvider: ContextProvider<Trait> = { + id: 'traitsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => Promise.resolve([{ name: 'test_trait', value: 'test_value', id: 'trait1' }]), + }, + }; + const codeSnippetsProvider: ContextProvider<CodeSnippet> = { + id: 'codeSnippetsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: (): Promise<CodeSnippet[]> => + Promise.resolve([ + { + uri: 'file:///something.ts', + value: dedent` + function foo() { + return 1; + } + `, + id: 'cs1', + }, + { + uri: 'file:///somethingElse.ts', + value: dedent` + function bar() { + return 'two'; + } + `, + id: 'cs2', + origin: 'update', + }, + ]), + }, + }; + // Register the documents for content exclusion + const contextProviderRegistry = accessor.get(ICompletionsContextProviderRegistryService); + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument('file:///something.ts', 'typescript', 'does not matter'); + + contextProviderRegistry.registerContextProvider(traitsContextProvider); + contextProviderRegistry.registerContextProvider(codeSnippetsProvider); + + const prompt = await invokePromptFactory(); + + const expectedTelemetry: ContextProviderTelemetry[] = [ + { + providerId: 'traitsProvider', + resolution: 'full', + resolutionTimeMs: -1, + usage: 'full', + matched: true, + numResolvedItems: 1, + numUsedItems: 1, + numPartiallyUsedItems: 0, + usageDetails: [{ id: 'trait1', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }], + }, + { + providerId: 'codeSnippetsProvider', + resolution: 'full', + resolutionTimeMs: -1, + usage: 'full', + matched: true, + numResolvedItems: 2, + numUsedItems: 2, + numPartiallyUsedItems: 0, + usageDetails: [ + { id: 'cs1', usage: 'full', expectedTokens: 13, actualTokens: 13, type: 'CodeSnippet' }, + { id: 'cs2', usage: 'full', expectedTokens: 13, actualTokens: 13, type: 'CodeSnippet', origin: 'update' }, + ], + }, + ]; + + assert.deepStrictEqual(prompt.type, 'prompt'); + assert.deepStrictEqual( + prompt.contextProvidersTelemetry?.map(pt => { + pt.resolutionTimeMs = -1; + return pt; + }), + expectedTelemetry + ); + }); + + test('Test only sanctioned traits are included in telemetry', async function () { + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = 'traitsProvider'; + + const traitsProvider: ContextProvider<Trait> = { + id: 'traitsProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => + Promise.resolve([ + { name: 'trait1', value: 'value1' }, + { name: 'TargetFrameworks', value: 'framework value' }, + { name: 'trait2', value: 'value2' }, + { name: 'LanguageVersion', value: 'language version' }, + ]), + }, + }; + const contextProviderRegistry = accessor.get(ICompletionsContextProviderRegistryService); + contextProviderRegistry.registerContextProvider(traitsProvider); + + const { reporter } = await withInMemoryTelemetry(accessor, async _ => { + const response = await invokePromptFactory(); + assert.deepStrictEqual(response.type, 'prompt'); + assert.deepStrictEqual( + response.prompt.prefix, + dedent` + // Path: basename + // Consider this related information: + // trait1: value1 + // TargetFrameworks: framework value + // trait2: value2 + // LanguageVersion: language version + ` + `\n${longPrefix}\nfunction f` + ); + }); + + // the event should only contains sanctioned trait with expected property names. + assert.strictEqual(reporter.hasEvent, true); + assert.strictEqual(reporter.events.length, 1); + + assert.strictEqual(reporter.events[0].name, 'contextProvider.traits'); + assert.strictEqual(reporter.events[0].properties['targetFrameworks'], 'framework value'); + assert.strictEqual(reporter.events[0].properties['languageVersion'], 'language version'); + assert.strictEqual(reporter.events[0].properties['languageId'], 'typescript'); + + assert.strictEqual(reporter.events[0].properties['trait1'], undefined); + assert.strictEqual(reporter.events[0].properties['trait2'], undefined); + + assert.strictEqual(reporter.hasException, false); + }); +}); + +class MockRecentEditsProvider extends FullRecentEditsProvider { + testUpdateRecentEdits(docId: string, newContents: string): void { + return this.updateRecentEdits(docId, newContents); + } +} + +export class CompletionsMutableObservableWorkspace extends MutableObservableWorkspace implements ICompletionsObservableWorkspace { + declare _serviceBrand: undefined; +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/codeSnippets.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/codeSnippets.tsx new file mode 100644 index 0000000..98af334 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/codeSnippets.tsx @@ -0,0 +1,102 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { Chunk, ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { + CompletionRequestDocument, + isCompletionRequestData, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { addRelativePathToCodeSnippets, CodeSnippetWithRelativePath } from '../contextProviders/codeSnippets'; +import { CodeSnippetWithId } from '../contextProviders/contextItemSchemas'; + +type CodeSnippetsProps = { + tdms: ICompletionsTextDocumentManagerService; +} & PromptElementProps; + +export const CodeSnippets = (props: CodeSnippetsProps, context: ComponentContext) => { + const [snippets, setSnippets] = context.useState<CodeSnippetWithId[]>(); + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + + context.useData(isCompletionRequestData, request => { + if (request.codeSnippets !== snippets) { + setSnippets(request.codeSnippets); + } + if (request.document.uri !== document?.uri) { + setDocument(request.document); + } + }); + + if (!snippets || snippets.length === 0 || !document) { + return; + } + + const codeSnippetsWithRelativePath = addRelativePathToCodeSnippets(props.tdms, snippets); + + // Snippets with the same URI should appear together as a single snippet. + const snippetsByUri = new Map<string, CodeSnippetWithRelativePath[]>(); + + for (const snippet of codeSnippetsWithRelativePath) { + const uri = snippet.relativePath ?? snippet.snippet.uri; + let groupedSnippets = snippetsByUri.get(uri); + if (groupedSnippets === undefined) { + groupedSnippets = []; + snippetsByUri.set(uri, groupedSnippets); + } + groupedSnippets.push(snippet); + } + + const codeSnippetChunks: { + chunkElements: CodeSnippetWithId[]; + importance: number; + uri: string; + }[] = []; + for (const [uri, snippets] of snippetsByUri.entries()) { + const validSnippets = snippets.filter(s => s.snippet.value.length > 0); + if (validSnippets.length > 0) { + codeSnippetChunks.push({ + chunkElements: validSnippets.map(s => s.snippet), + // The importance is the maximum importance of the snippets in this group. + importance: Math.max(...validSnippets.map(snippet => snippet.snippet.importance ?? 0)), + uri, + }); + } + } + + if (codeSnippetChunks.length === 0) { + return; + } + + // Sort by importance, with the most important first + codeSnippetChunks.sort((a, b) => b.importance - a.importance); + // Reverse the order so the most important snippet is last. Note, that we don't directly + // sort in ascending order to handle importance 0 correctly. + codeSnippetChunks.reverse(); + return codeSnippetChunks.map(chunk => { + const elements = []; + + elements.push( + <Text> + {`Compare ${chunk.chunkElements.length > 1 ? 'these snippets' : 'this snippet'} from ${chunk.uri}:`} + </Text> + ); + + chunk.chunkElements.forEach((element, index) => { + elements.push( + <Text source={element} key={element.id}> + {element.value} + </Text> + ); + if (chunk.chunkElements.length > 1 && index < chunk.chunkElements.length - 1) { + elements.push(<Text>---</Text>); + } + }); + + // TODO: change Chunk for KeepTogether + return <Chunk>{elements}</Chunk>; + }); +}; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/completionsContext.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/completionsContext.tsx new file mode 100644 index 0000000..d3bcac2 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/completionsContext.tsx @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +//** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { PromptElementProps, PromptSnapshotNode } from '../../../../prompt/src/components/components'; + +/** + * A component that marks the context part of the prompt + */ +export function CompletionsContext(props: PromptElementProps) { + return props.children; +} + +/** + * A component that marks the context part of the prompt that is stable across requests, + * and should be located earlier in the prompt to maximize cache hits. + */ +export function StableCompletionsContext(props: PromptElementProps) { + return props.children; +} + +/** + * A component that marks the context part of the prompt that is subject to change quickly across requests, + * and should be located further down in the prompt. + */ +export function AdditionalCompletionsContext(props: PromptElementProps) { + return props.children; +} + +export function isContextNode(node: PromptSnapshotNode): boolean { + return ( + node.name === CompletionsContext.name || + node.name === StableCompletionsContext.name || + node.name === AdditionalCompletionsContext.name + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/completionsPromptRenderer.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/completionsPromptRenderer.tsx new file mode 100644 index 0000000..f0893ea --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/completionsPromptRenderer.tsx @@ -0,0 +1,296 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { CancellationToken } from 'vscode-languageserver-protocol'; +import { + ComponentStatistics, + PromptOk, + PromptRenderer, + PromptRenderOptions, + PromptSnapshotNode, + StatusNotOk, +} from '../../../../prompt/src/components/components'; +import { defaultTransformers, SnapshotWalker, WalkContextTransformer } from '../../../../prompt/src/components/walker'; +import { commentBlockAsSingles, isShebangLine } from '../../../../prompt/src/languageMarker'; +import { getTokenizer, TokenizerName } from '../../../../prompt/src/tokenization'; +import { isContextNode } from './completionsContext'; +import { AfterCursor, BeforeCursor, CurrentFile } from './currentFile'; +import { ElidedBlock, makePrompt, WeightedBlock, WishlistElision } from './elision'; + +const TOKENS_RESERVED_FOR_SUFFIX_ENCODING = 5; + +export type CompletionsPromptOk = PromptOk & { + prefix: string; + prefixTokens: number; + suffix: string; + suffixTokens: number; + context: string[] | undefined; +}; +type CompletionsPrompt = CompletionsPromptOk | StatusNotOk; + +export interface CompletionsPromptRenderOptions extends PromptRenderOptions { + promptTokenLimit: number; + suffixPercent: number; + languageId: string; + delimiter?: string; +} + +export class CompletionsPromptRenderer implements PromptRenderer<CompletionsPrompt, CompletionsPromptRenderOptions> { + private renderId = 0; + + /** + * Function used to format the prefix blocks into a string. + * If implementing a renderer subclass, override this to control how the prefix is formatted, otherwise defaults to `makePrompt`. + */ + protected formatPrefix: (elidedBlocks: ElidedBlock[]) => string = makePrompt; + + /** + * Function used to format the context blocks into a string array. + * Context is optional, so leave this as `undefined` if you do not want to include context in the rendered prompt. + * If implementing a renderer subclass, override this to control how the context is formatted, otherwise defaults to `undefined`. + */ + protected formatContext: undefined | ((elidedBlocks: ElidedBlock[]) => string[]); + + render( + snapshot: PromptSnapshotNode, + options: CompletionsPromptRenderOptions, + cancellationToken?: CancellationToken + ): CompletionsPrompt { + const id = this.renderId++; + const renderStart = performance.now(); + try { + if (cancellationToken?.isCancellationRequested) { + return { status: 'cancelled' }; + } + // Default options + const delimiter = options.delimiter ?? ''; + const tokenizer = options.tokenizer ?? TokenizerName.o200k; + // Process the snapshot to get the prefix and suffix and adjust the token limits accordingly + const { prefixBlocks, suffixBlock, componentStatistics } = this.processSnapshot( + snapshot, + delimiter, + options.languageId + ); + + const { prefixTokenLimit, suffixTokenLimit } = this.getPromptLimits(suffixBlock, options); + const elisionStart = performance.now(); + const elisionStrategy = new WishlistElision(); + // The first element is always the suffix + const { + blocks: [elidedSuffix, ...elidedPrefix], + } = elisionStrategy.elide( + prefixBlocks, + prefixTokenLimit, + suffixBlock, + suffixTokenLimit, + getTokenizer(tokenizer) + ); + const elisionEnd = performance.now(); + + const prefix = this.formatPrefix(elidedPrefix); + const context = this.formatContext ? this.formatContext(elidedPrefix) : undefined; + const suffix = elidedSuffix.elidedValue; + const prefixTokens = elidedPrefix.reduce((acc, block) => acc + block.elidedTokens, 0); + + componentStatistics.push(...computeComponentStatistics([...elidedPrefix, elidedSuffix])); + return { + prefix, + prefixTokens, + suffix, + suffixTokens: elidedSuffix.elidedTokens, + context, + status: 'ok', + metadata: { + renderId: id, + rendererName: 'c', + tokenizer: tokenizer, + elisionTimeMs: elisionEnd - elisionStart, + renderTimeMs: performance.now() - renderStart, + componentStatistics, + updateDataTimeMs: componentStatistics.reduce( + (acc, component) => acc + (component.updateDataTimeMs ?? 0), + 0 + ), + }, + }; + } catch (e) { + return { status: 'error', error: e as Error }; + } + } + + // Defaults are hardcoded for now, but we can use EXP flags like PromptOptions does + // by passing the context + private getPromptLimits(suffixBlock: WeightedBlock | undefined, options: CompletionsPromptRenderOptions) { + const suffix = suffixBlock?.value ?? ''; + + let availableTokens = options.promptTokenLimit; + const suffixPercent = options.suffixPercent; + + if (suffix.length === 0 || suffixPercent === 0) { + return { prefixTokenLimit: availableTokens, suffixTokenLimit: 0 }; + } + + // If there is a suffix, we need to reserve some tokens for the suffix encoding + availableTokens = suffix.length > 0 ? availableTokens - TOKENS_RESERVED_FOR_SUFFIX_ENCODING : availableTokens; + + const suffixTokenLimit = Math.ceil(availableTokens * (suffixPercent / 100)); + const prefixTokenLimit = availableTokens - suffixTokenLimit; + + return { + prefixTokenLimit, + suffixTokenLimit, + }; + } + + protected processSnapshot( + snapshot: PromptSnapshotNode, + delimiter: string, + languageId: string + ): { + prefixBlocks: WeightedBlock[]; + suffixBlock: WeightedBlock; + componentStatistics: ComponentStatistics[]; + } { + const prefixBlocks: WeightedBlock[] = []; + const suffixBlocks: WeightedBlock[] = []; + const componentStatistics: ComponentStatistics[] = []; + // Store the status of the required nodes + let foundDocument = false; + + const walker = new SnapshotWalker(snapshot, transformers); + walker.walkSnapshot((node, _parent, context) => { + if (node === snapshot) { + return true; + } + + // Check for the presence of required node + if (node.name === CurrentFile.name) { + foundDocument = true; + } + + if (node.statistics.updateDataTimeMs && node.statistics.updateDataTimeMs > 0) { + componentStatistics.push({ + componentPath: node.path, + updateDataTimeMs: node.statistics.updateDataTimeMs, + }); + } + + if (node.value === undefined || node.value === '') { + // No need to process this node as it only adds whitespace + return true; + } + + const chunks = context.chunks as Set<string> | undefined; + if (context.type === 'suffix') { + // Everything after the cursor is part of the suffix + suffixBlocks.push({ + value: normalizeLineEndings(node.value), + type: 'suffix', + weight: context.weight as number, + componentPath: node.path, + nodeStatistics: node.statistics, + chunks, + source: context.source, + }); + } else { + // Add a delimiter for all nodes, that are not the beforeCursor if not already present + const nodeValueWithDelimiter = node.value.endsWith(delimiter) ? node.value : node.value + delimiter; + let value = nodeValueWithDelimiter; + if (context.type === 'prefix') { + value = node.value; + } else if (isShebangLine(node.value)) { + value = nodeValueWithDelimiter; + } else { + value = commentBlockAsSingles(nodeValueWithDelimiter, languageId); + } + prefixBlocks.push({ + type: context.type === 'prefix' ? 'prefix' : 'context', + value: normalizeLineEndings(value), + weight: context.weight as number, + componentPath: node.path, + nodeStatistics: node.statistics, + chunks, + source: context.source, + }); + } + return true; + }); + + if (!foundDocument) { + throw new Error(`Node of type ${CurrentFile.name} not found`); + } + if (suffixBlocks.length > 1) { + throw new Error(`Only one suffix is allowed`); + } + + const suffixBlock: WeightedBlock = + suffixBlocks.length === 1 + ? suffixBlocks[0] + : { + componentPath: '', + value: '', + weight: 1, + nodeStatistics: {}, + type: 'suffix', + }; + return { prefixBlocks, suffixBlock, componentStatistics }; + } +} + +export const transformers: WalkContextTransformer[] = [ + ...defaultTransformers(), + // Context transformer + (node, _, context) => { + if (isContextNode(node)) { + return { ...context, type: 'context' }; + } + return context; + }, + // Prefix transformer + (node, _, context) => { + if (node.name === BeforeCursor.name) { + return { + ...context, + type: 'prefix', + }; + } + return context; + }, + // Suffix transformer + (node, _, context) => { + if (node.name === AfterCursor.name) { + return { + ...context, + type: 'suffix', + }; + } + return context; + }, +]; + +function computeComponentStatistics(elidedBlocks: ElidedBlock[]) { + return elidedBlocks.map(block => { + const result: ComponentStatistics = { + componentPath: block.componentPath, + }; + if (block.tokens !== 0) { + result.expectedTokens = block.tokens; + result.actualTokens = block.elidedTokens; + } + if (block.nodeStatistics.updateDataTimeMs !== undefined) { + result.updateDataTimeMs = block.nodeStatistics.updateDataTimeMs; + } + if (block.source) { + result.source = block.source; + } + return result; + }); +} + +export function normalizeLineEndings(text: string) { + return text.replace(/\r\n?/g, '\n'); +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/contextProviderBridge.ts b/completions-sample-code/vscode-node/lib/src/prompt/components/contextProviderBridge.ts new file mode 100644 index 0000000..0c35655 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/contextProviderBridge.ts @@ -0,0 +1,71 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../../../util/common/services'; +import { CancellationToken } from '../../../../types/src'; +import { CompletionState } from '../../completionState'; +import { LRUCacheMap } from '../../helpers/cache'; +import { TelemetryWithExp } from '../../telemetry'; +import { ICompletionsContextProviderRegistryService, ResolvedContextItem } from '../contextProviderRegistry'; + +export const ICompletionsContextProviderBridgeService = createServiceIdentifier<ICompletionsContextProviderBridgeService>('ICompletionsContextProviderBridgeService'); +export interface ICompletionsContextProviderBridgeService { + readonly _serviceBrand: undefined; + schedule( + completionState: CompletionState, + completionId: string, + opportunityId: string, + telemetryData: TelemetryWithExp, + cancellationToken?: CancellationToken, + options?: { data?: unknown } + ): void; + + resolution(id: string): Promise<ResolvedContextItem[]>; +} + +export class ContextProviderBridge implements ICompletionsContextProviderBridgeService { + declare _serviceBrand: undefined; + private scheduledResolutions = new LRUCacheMap<string, Promise<ResolvedContextItem[]>>(25); + + constructor(@ICompletionsContextProviderRegistryService private readonly contextProviderRegistry: ICompletionsContextProviderRegistryService) { } + + schedule( + completionState: CompletionState, + completionId: string, + opportunityId: string, + telemetryData: TelemetryWithExp, + cancellationToken?: CancellationToken, + options?: { data?: unknown } + ) { + const { textDocument, originalPosition, originalOffset, originalVersion, editsWithPosition } = completionState; + + const resolutionPromise = this.contextProviderRegistry.resolveAllProviders( + completionId, + opportunityId, + { + uri: textDocument.uri, + languageId: textDocument.detectedLanguageId, + version: originalVersion, + offset: originalOffset, + position: originalPosition, + proposedEdits: editsWithPosition.length > 0 ? editsWithPosition : undefined, + }, + telemetryData, + cancellationToken, + options?.data + ); + + this.scheduledResolutions.set(completionId, resolutionPromise); + // intentionally not awaiting to avoid blocking + } + + async resolution(id: string): Promise<ResolvedContextItem[]> { + const resolutionPromise = this.scheduledResolutions.get(id); + if (resolutionPromise) { + return await resolutionPromise; + } + return []; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/currentFile.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/currentFile.tsx new file mode 100644 index 0000000..03eaed0 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/currentFile.tsx @@ -0,0 +1,220 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { Position } from 'vscode-languageserver-protocol'; +import { ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { DEFAULT_SUFFIX_MATCH_THRESHOLD } from '../../../../prompt/src/prompt'; +import { findEditDistanceScore } from '../../../../prompt/src/suffixMatchCriteria'; +import { getTokenizer, TokenizerName } from '../../../../prompt/src/tokenization'; +import { + CompletionRequestDocument, + isCompletionRequestData, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; + +/** The maximum number of tokens that is used for calculate edit distance. */ +export const MAX_EDIT_DISTANCE_LENGTH = 50; + +function approximateMaxCharacters(maxPromptLength: number): number { + const maxCharsInPrompt = maxPromptLength * 4; // approximate 4 chars per token + const compensation = maxPromptLength * 0.1; // 10% overflow to compensate the token approximation + return Math.floor(maxCharsInPrompt + compensation); +} + +/** + * A required component for the CompletionsPromptRenderer. It represents the document and position where completions should be shown. + */ +export function CurrentFile(_props: PromptElementProps, context: ComponentContext) { + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + const [position, setPosition] = context.useState<Position>(); + const [maxPromptLength, setMaxPromptLength] = context.useState<number>(0); + const [suffixMatchThreshold, setSuffixMatchThreshold] = context.useState<number>(); + const [tokenizer, setTokenizer] = context.useState<TokenizerName>(); + + context.useData(isCompletionRequestData, request => { + const requestDocument = request.document; + if (request.document.uri !== document?.uri || requestDocument.getText() !== document?.getText()) { + setDocument(requestDocument); + } + + if (request.position !== position) { + setPosition(request.position); + } + + if (request.suffixMatchThreshold !== suffixMatchThreshold) { + setSuffixMatchThreshold(request.suffixMatchThreshold); + } + + if (request.maxPromptTokens !== maxPromptLength) { + setMaxPromptLength(request.maxPromptTokens); + } + + if (request.tokenizer !== tokenizer) { + setTokenizer(request.tokenizer); + } + }); + + const maxCharacters = approximateMaxCharacters(maxPromptLength); + return ( + <> + <BeforeCursor document={document} position={position} maxCharacters={maxCharacters} /> + <AfterCursor + document={document} + position={position} + suffixMatchThreshold={suffixMatchThreshold} + maxCharacters={maxCharacters} + tokenizer={tokenizer} + /> + </> + ); +} + +export function BeforeCursor(props: { + document: CompletionRequestDocument | undefined; + position: Position | undefined; + maxCharacters: number; +}) { + if (props.document === undefined || props.position === undefined) { + return <Text />; + } + + let text = props.document.getText({ start: { line: 0, character: 0 }, end: props.position }); + if (text.length > props.maxCharacters) { + text = text.slice(-props.maxCharacters); + } + return <Text>{text}</Text>; +} + +export function AfterCursor( + props: { + document: CompletionRequestDocument | undefined; + position: Position | undefined; + maxCharacters: number; + suffixMatchThreshold?: number; + tokenizer?: TokenizerName; + }, + context: ComponentContext +) { + const [cachedSuffix, setCachedSuffix] = context.useState<string>(''); + + if (props.document === undefined || props.position === undefined) { + return <Text />; + } + + let suffix = props.document.getText({ + start: props.position, + end: { line: Number.MAX_VALUE, character: Number.MAX_VALUE }, + }); + if (suffix.length > props.maxCharacters) { + suffix = suffix.slice(0, props.maxCharacters); + } + + // Start the suffix at the beginning of the next line. This allows for consistent reconciliation of trailing punctuation. + const trimmedSuffix = suffix.replace(/^.*/, '').trimStart(); + if (trimmedSuffix === '') { + return <Text />; + } + + // Cache hit + if (cachedSuffix === trimmedSuffix) { + return <Text>{cachedSuffix}</Text>; + } + + let suffixToUse = trimmedSuffix; + if (cachedSuffix !== '') { + const tokenizer = getTokenizer(props.tokenizer); + const firstSuffixTokens = tokenizer.takeFirstTokens(trimmedSuffix, MAX_EDIT_DISTANCE_LENGTH); + // Check if the suffix is similar to the cached suffix. + // See docs/suffix_caching.md for some background about why we do this. + if (firstSuffixTokens.tokens.length > 0) { + // Calculate the distance between the computed and cached suffixed using Levenshtein distance. + // Only compare the first MAX_EDIT_DISTANCE_LENGTH tokens to speed up. + const dist = findEditDistanceScore( + firstSuffixTokens.tokens, + tokenizer.takeFirstTokens(cachedSuffix, MAX_EDIT_DISTANCE_LENGTH).tokens + )?.score; + if ( + 100 * dist < + (props.suffixMatchThreshold ?? DEFAULT_SUFFIX_MATCH_THRESHOLD) * firstSuffixTokens.tokens.length + ) { + suffixToUse = cachedSuffix; + } + } + } + + // Only set the suffix if it's different from the cached one, otherwise we rerender this component all the time + if (suffixToUse !== cachedSuffix) { + setCachedSuffix(suffixToUse); + } + + return <Text>{suffixToUse}</Text>; +} + +export function DocumentPrefix(_props: PromptElementProps, context: ComponentContext) { + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + const [position, setPosition] = context.useState<Position>(); + const [maxPromptLength, setMaxPromptLength] = context.useState<number>(0); + + context.useData(isCompletionRequestData, request => { + const requestDocument = request.document; + if (request.document.uri !== document?.uri || requestDocument.getText() !== document?.getText()) { + setDocument(requestDocument); + } + + if (request.position !== position) { + setPosition(request.position); + } + + if (request.maxPromptTokens !== maxPromptLength) { + setMaxPromptLength(request.maxPromptTokens); + } + }); + + const maxCharacters = approximateMaxCharacters(maxPromptLength); + + return <BeforeCursor document={document} position={position} maxCharacters={maxCharacters} />; +} + +export function DocumentSuffix(_props: PromptElementProps, context: ComponentContext) { + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + const [position, setPosition] = context.useState<Position>(); + const [maxPromptLength, setMaxPromptLength] = context.useState<number>(0); + const [suffixMatchThreshold, setSuffixMatchThreshold] = context.useState<number>(); + const [tokenizer, setTokenizer] = context.useState<TokenizerName>(); + + context.useData(isCompletionRequestData, request => { + const requestDocument = request.document; + if (request.document.uri !== document?.uri || requestDocument.getText() !== document?.getText()) { + setDocument(requestDocument); + } + + if (request.position !== position) { + setPosition(request.position); + } + + if (request.suffixMatchThreshold !== suffixMatchThreshold) { + setSuffixMatchThreshold(request.suffixMatchThreshold); + } + + if (request.maxPromptTokens !== maxPromptLength) { + setMaxPromptLength(request.maxPromptTokens); + } + + if (request.tokenizer !== tokenizer) { + setTokenizer(request.tokenizer); + } + }); + const maxCharacters = approximateMaxCharacters(maxPromptLength); + return ( + <AfterCursor + document={document} + position={position} + suffixMatchThreshold={suffixMatchThreshold} + maxCharacters={maxCharacters} + tokenizer={tokenizer} + /> + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/diagnostics.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/diagnostics.tsx new file mode 100644 index 0000000..d9c0c80 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/diagnostics.tsx @@ -0,0 +1,118 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { DiagnosticSeverity, type Diagnostic } from 'vscode'; +import { Chunk, ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { normalizeLanguageId } from '../../../../prompt/src/prompt'; +import type { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { + CompletionRequestData, + isCompletionRequestData, + type CompletionRequestDocument, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { type DiagnosticBagWithId } from '../contextProviders/contextItemSchemas'; + + +function getCode(diagnostic: Diagnostic): string | undefined { + if (diagnostic.code === undefined) { + return undefined; + } + if (typeof diagnostic.code === 'string') { + return diagnostic.code; + } + if (typeof diagnostic.code === 'number') { + return diagnostic.code.toString(); + } + if (typeof diagnostic.code === 'object' && diagnostic.code !== null && diagnostic.code.value) { + return diagnostic.code.value.toString(); + } + return undefined; +} + +function getRelativePath(tdm: ICompletionsTextDocumentManagerService, item: DiagnosticBagWithId): string { + return tdm.getRelativePath({ uri: item.uri.toString() }) ?? item.uri.path; +} + +type DiagnosticsProps = { + tdms: ICompletionsTextDocumentManagerService; +} & PromptElementProps; + + +export const Diagnostics = (props: DiagnosticsProps, context: ComponentContext) => { + const [diagnostics, setDiagnostics] = context.useState<DiagnosticBagWithId[]>(); + const [languageId, setLanguageId] = context.useState<string>(); + const [position, setPosition] = context.useState<{ line: number; character: number }>(); + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + + + context.useData(isCompletionRequestData, (data: CompletionRequestData) => { + if (data.diagnostics !== diagnostics) { + setDiagnostics(data.diagnostics); + } + + const normalizedLanguageId = normalizeLanguageId(data.document.detectedLanguageId); + if (normalizedLanguageId !== languageId) { + setLanguageId(normalizedLanguageId); + } + + if (data.position !== position) { + setPosition(data.position); + } + + if (data.document.uri !== document?.uri) { + setDocument(data.document); + } + }); + + if (!diagnostics || diagnostics.length === 0 || !languageId) { + return; + } + const validChunks = diagnostics.filter(diagnostic => diagnostic.values.length > 0); + if (validChunks.length === 0) { + return; + } + + // Sort by importance, with the most important first + validChunks.sort((a, b) => (b.importance ?? 0) - (a.importance ?? 0)); + // Reverse the order so the most important snippet is last. Note, that we don't directly + // sort in ascending order to handle importance 0 correctly. + validChunks.reverse(); + + return validChunks.map(diagnosticBag => { + const elements = []; + elements.push( + <Text key={diagnosticBag.id} source={diagnosticBag}> + {`Consider the following ${languageId} diagnostics from ${getRelativePath(props.tdms, diagnosticBag)}:`} + </Text> + ); + let values: Diagnostic[] = diagnosticBag.values; + if (document !== undefined && document.uri.toString() === diagnosticBag.uri.toString() && position !== undefined) { + // Create a copy of the diagnostics to avoid mutating the original array in the context item in case it is used elsewhere. + values = diagnosticBag.values.slice(); + values.sort((a, b) => { + const aDist = Math.abs(a.range.start.line - position.line); + const bDist = Math.abs(b.range.start.line - position.line); + return aDist - bDist; + }); + } + values.forEach(diagnostic => { + let codeStr = ''; + const code = getCode(diagnostic); + if (code !== undefined) { + const source = diagnostic.source ? diagnostic.source.toUpperCase() : ''; + codeStr = ` ${source}${code}`; + } + const start = diagnostic.range.start; + elements.push( + <Text> + {`${start.line + 1}:${start.character + 1} - ${DiagnosticSeverity[diagnostic.severity].toLowerCase()}${codeStr}: ${diagnostic.message}`} + </Text> + ); + }); + return <Chunk>{elements}</Chunk>; + }); +}; \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/elision.ts b/completions-sample-code/vscode-node/lib/src/prompt/components/elision.ts new file mode 100644 index 0000000..562939c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/elision.ts @@ -0,0 +1,381 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PromptSnapshotNodeStatistics } from '../../../../prompt/src/components/components'; +import { Tokenizer } from '../../../../prompt/src/tokenization'; + +export interface WeightedBlock { + /** + * Paths use a syntax similar to JSON path, but with a few differences: + * - Fragments, string and number components are ignored + * - The same identifier can exist at the same level, and is represented as an array index ([i]) + * For example this prompts: + * <> + * <ComponentA /> + * <ComponentB /> + * <ComponentA /> + * </> + * Would have the following paths: + * $.ComponentA[0] + * $.ComponentB + * $.ComponentA[1] + */ + componentPath: string; + type: 'prefix' | 'suffix' | 'context'; + // The original text value of the block + value: string; + weight: number; + index?: number; // Optional block index, used to group context items + nodeStatistics: PromptSnapshotNodeStatistics; + chunks?: Set<string>; + source?: unknown; +} +interface ElidableBlock extends WeightedBlock { + // The number of tokens of the original value + tokens: number; + markedForRemoval: boolean; +} + +interface LineWithPathAndTokens { + line: string; + componentPath: string; + tokens: number; +} + +interface PrefixElidableBlock extends ElidableBlock { + originalIndex: number; + lines: LineWithPathAndTokens[]; +} + +export interface ElidedBlock extends WeightedBlock { + tokens: number; + elidedValue: string; + elidedTokens: number; +} + +interface ElisionStrategy { + elide( + prefixBlocks: WeightedBlock[], + prefixTokenLimit: number, + suffixBlock: WeightedBlock, + suffixTokenLimit: number, + tokenizer: Tokenizer + ): { blocks: ElidedBlock[]; cycles: number }; +} + +/** + * The wishlist strategy does a two-pass elision, based on prompt/src/wishlist.ts + * - Removes blocks (of lines) with the lowest weight, ignoring blocks with a weight of 1. + * - Adjust the total token count to fit within the limit line by line, top to bottom. + * + * Notice the extra `suffix*` arguments in the constructor and elide method. + */ +export class WishlistElision implements ElisionStrategy { + elide( + prefixBlocks: WeightedBlock[], + prefixTokenLimit: number, + suffixBlock: WeightedBlock, + suffixTokenLimit: number, + tokenizer: Tokenizer + ) { + if (prefixTokenLimit <= 0) { + throw new Error('Prefix limit must be greater than 0'); + } + + const [elidablePrefixBlocks, maxPrefixTokens] = this.preparePrefixBlocks(prefixBlocks, tokenizer); + const { elidedSuffix, adjustedPrefixTokenLimit } = this.elideSuffix( + suffixBlock, + suffixTokenLimit, + prefixTokenLimit, + maxPrefixTokens, + tokenizer + ); + const elidedPrefix = this.elidePrefix( + elidablePrefixBlocks, + adjustedPrefixTokenLimit, + maxPrefixTokens, + tokenizer + ); + + return { blocks: [elidedSuffix, ...elidedPrefix], cycles: 1 }; + } + + private preparePrefixBlocks(blocks: WeightedBlock[], tokenizer: Tokenizer): [PrefixElidableBlock[], number] { + let maxPrefixTokens = 0; + // Create a set to keep track of component paths + const componentPaths = new Set<string>(); + + const elidableBlocks = blocks.map((block, index) => { + let blockTokens = 0; + // Update the total tokens by approximating the length of a block with the sum + // of the lengths of its lines. Lines are split by newlines, and the newline + // value is kept together with the line (and hence counted as a token). + const blockLines = block.value.split(/([^\n]*\n+)/).filter(l => l !== ''); + const processedBlockLines = blockLines.map(line => { + const tokens = tokenizer.tokenLength(line); + blockTokens += tokens; + maxPrefixTokens += tokens; + return { line, componentPath: block.componentPath, tokens }; + }); + // Check if the component path is unique + const componentPath = block.componentPath; + if (componentPaths.has(componentPath)) { + throw new Error(`Duplicate component path in prefix blocks: ${componentPath}`); + } + componentPaths.add(componentPath); + return { + ...block, + tokens: blockTokens, + markedForRemoval: false, + originalIndex: index, + lines: processedBlockLines, + }; + }); + + return [elidableBlocks, maxPrefixTokens]; + } + + /** + * Special handling for the suffix, adapted from PromptWishlist.fulfill + * Some behaviors are different from the original implementation: + * - If the token limit is less than the edit distance, we don't error but just return the first tokens of the new suffix. + * - When using the cached suffix, we check and enforce the limit. + * - Remaining tokens are returned and handled by the caller, so we don't need to check the prefix nor modify limits in place. + */ + private elideSuffix( + suffixBlock: WeightedBlock, + suffixTokenLimit: number, + prefixTokenLimit: number, + maxPrefixTokens: number, + tokenizer: Tokenizer + ) { + const suffix = suffixBlock.value; + if (suffix.length === 0 || suffixTokenLimit <= 0) { + const elidedSuffix: ElidedBlock = { + ...suffixBlock, + tokens: 0, + elidedValue: '', + elidedTokens: 0, + }; + return { + elidedSuffix, + adjustedPrefixTokenLimit: prefixTokenLimit + Math.max(0, suffixTokenLimit), + }; + } + + // Check the maximum (approximate) length of the prefix. + // If everything fits, we give the remaining budget to the suffix instead. + if (maxPrefixTokens < prefixTokenLimit) { + suffixTokenLimit = suffixTokenLimit + (prefixTokenLimit - maxPrefixTokens); + prefixTokenLimit = maxPrefixTokens; + } + + const shortenedSuffix = tokenizer.takeFirstTokens(suffix, suffixTokenLimit); + const elidedSuffix: ElidedBlock = { + ...suffixBlock, + // Update the original value and tokens + value: suffix, + tokens: tokenizer.tokenLength(suffix), + elidedValue: shortenedSuffix.text, + elidedTokens: shortenedSuffix.tokens.length, + }; + + return { + elidedSuffix, + adjustedPrefixTokenLimit: prefixTokenLimit + Math.max(0, suffixTokenLimit - shortenedSuffix.tokens.length), + }; + } + + private elidePrefix( + elidablePrefixBlocks: PrefixElidableBlock[], + tokenLimit: number, + maxPrefixTokens: number, + tokenizer: Tokenizer + ): ElidedBlock[] { + const prefixBlocks = this.removeLowWeightPrefixBlocks(elidablePrefixBlocks, tokenLimit, maxPrefixTokens); + + // The nodes that are not marked for removal are split into lines, but we keep + // track of the block they came from + const prefixLines = prefixBlocks.filter(block => !block.markedForRemoval).flatMap(block => block.lines); + + if (prefixLines.length === 0) { + return []; + } + + const [trimmedLines, prefixTokens] = this.trimPrefixLinesToFit(prefixLines, tokenLimit, tokenizer); + // Populate the final elidable blocks + let currentPrefixTokens = prefixTokens; + return prefixBlocks.map(block => { + if (block.markedForRemoval) { + // Try to re-include blocks if there's space left and they are not part of a chunk + if (currentPrefixTokens + block.tokens <= tokenLimit && !block.chunks) { + // This is an approximation, but we don't want to add more token operations. + // In the wishlist, this is done using the priority list, but for simplicity we just + // do it in order. + currentPrefixTokens += block.tokens; + return { ...block, elidedValue: block.value, elidedTokens: block.tokens }; + } + return { ...block, elidedValue: '', elidedTokens: 0 }; + } + + const elidedValue = trimmedLines + .filter(l => l.componentPath === block.componentPath && l.line !== '') + .map(l => l.line) + .join(''); + let elidedTokens = block.tokens; + if (elidedValue !== block.value) { + elidedTokens = elidedValue !== '' ? tokenizer.tokenLength(elidedValue) : 0; + } + + return { ...block, elidedValue, elidedTokens }; + }); + } + + /** + * Marks blocks for removal based on their weight and the total token limit. + * If a block has a chunk identifier, all blocks with the same chunk will be removed together. + * Blocks with a weight of 1 are protected from removal. + */ + private removeLowWeightPrefixBlocks( + elidablePrefixBlocks: PrefixElidableBlock[], + tokenLimit: number, + maxPrefixTokens: number + ): PrefixElidableBlock[] { + let totalPrefixTokens = maxPrefixTokens; + + // Sort the blocks by weight ascending + elidablePrefixBlocks.sort((a, b) => a.weight - b.weight); + // Remove blocks with the lowest weight until total tokens are within the limit + // If a block has a weight of 1, it is skipped in this step + for (const block of elidablePrefixBlocks) { + if (totalPrefixTokens <= tokenLimit) { break; } + if (block.weight === 1) { continue; } + + // If block has a chunk that's already been processed, skip it + if (block.chunks && block.markedForRemoval) { continue; } + + if (block.chunks && block.chunks.size > 0) { + // Mark all blocks with the same chunk for removal + for (const relatedBlock of elidablePrefixBlocks) { + if ( + !relatedBlock.markedForRemoval && + relatedBlock.chunks && + // For nested chunks: if removing outer chunk, remove all inner chunks + // by checking if the related block contains ALL chunk IDs from current block + [...block.chunks].every(id => relatedBlock.chunks?.has(id)) + ) { + relatedBlock.markedForRemoval = true; + totalPrefixTokens -= relatedBlock.tokens; + } + } + } else { + // Regular case: just mark this block for removal + block.markedForRemoval = true; + totalPrefixTokens -= block.tokens; + } + } + + // Sort the nodes by their original index + return elidablePrefixBlocks.sort((a, b) => a.originalIndex - b.originalIndex); + } + + private trimPrefixLinesToFit( + linesWithComponentPath: LineWithPathAndTokens[], + tokenLimit: number, + tokenizer: Tokenizer + ): [LineWithPathAndTokens[], number] { + let currentPrefixTokens = 0; + + // Create a new array to store lines that fit within the limit + const fittingLines: typeof linesWithComponentPath = []; + + // Iterate from the end of the array + for (let i = linesWithComponentPath.length - 1; i >= 0; i--) { + const currentLine = linesWithComponentPath[i]; + const lineTokens = currentLine.tokens; + + // Check if adding this line would exceed the limit + if (currentPrefixTokens + lineTokens <= tokenLimit) { + fittingLines.unshift(currentLine); // Add to front to maintain order + currentPrefixTokens += lineTokens; + } else { + break; // Stop once we exceed the limit + } + } + + if (fittingLines.length === 0) { + // This can still mean that the last line (the cursor line) is too long. + // So we try to fit the last line up to the limit. + const lastLine = linesWithComponentPath[linesWithComponentPath.length - 1]; + if (lastLine && lastLine.line.length > 0) { + const prompt = tokenizer.takeLastTokens(lastLine.line, tokenLimit); + fittingLines.push({ + line: prompt.text, + componentPath: lastLine.componentPath, + tokens: prompt.tokens.length, + }); + return [fittingLines, prompt.tokens.length]; + } + + const errorMsg = `Cannot fit prefix within limit of ${tokenLimit} tokens`; + throw new Error(errorMsg); + } + return [fittingLines, currentPrefixTokens]; + } +} + +export function makePrompt(elidedBlocks: ElidedBlock[]): string { + return elidedBlocks.map(block => block.elidedValue).join(''); +} + +export function makePrefixPrompt(elidedBlocks: ElidedBlock[]): string { + return elidedBlocks + .filter(b => b.type === 'prefix') + .map(block => block.elidedValue) + .join(''); +} + +/** + * Return context items grouped in blocks reflecting the prompt structure. + */ +export function makeContextPrompt(elidedBlocks: ElidedBlock[]): string[] { + if (elidedBlocks.length === 0) { + return []; + } + + // Group context items by index + const contextGroups = new Map<number, string[]>(); + for (const block of elidedBlocks) { + // Only consider context blocks with an index + if (block.type === 'context' && block.index !== undefined) { + // Initialize the group + if (!contextGroups.has(block.index)) { + contextGroups.set(block.index, []); + } + // Add the trimmed value + const trimmed = block.elidedValue.trim(); + if (trimmed.length > 0) { + contextGroups.get(block.index)!.push(trimmed); + } + } + } + + const maxIndex = Math.max(...Array.from(contextGroups.keys()), -1); + + // Create context blocks + const contextBlocks = []; + for (let i = 0; i <= maxIndex; i++) { + const group = contextGroups.get(i); + if (group && group.length > 0) { + const value = group.join('\n').trim(); + contextBlocks.push(value); + } else { + // If there are no items for this index, add an empty string to maintain ordering + contextBlocks.push(''); + } + } + + return contextBlocks; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/marker.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/marker.tsx new file mode 100644 index 0000000..d34a8e3 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/marker.tsx @@ -0,0 +1,52 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { getLanguageMarker, getPathMarker } from '../../../../prompt/src/languageMarker'; +import { DocumentInfo } from '../../../../prompt/src/prompt'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { + CompletionRequestDocument, + isCompletionRequestData, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; + +type DocumentMarkerProps = { + tdms: ICompletionsTextDocumentManagerService; +} & PromptElementProps; + +export const DocumentMarker = (props: DocumentMarkerProps, context: ComponentContext) => { + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + + context.useData(isCompletionRequestData, request => { + if (request.document.uri !== document?.uri) { + setDocument(request.document); + } + }); + + if (document) { + const relativePath = props.tdms.getRelativePath(document); + const docInfo: DocumentInfo = { + uri: document.uri, + source: document.getText(), + relativePath, + languageId: document.detectedLanguageId, + }; + const notebook = props.tdms.findNotebook(document); + if (docInfo.relativePath && !notebook) { + return <PathMarker docInfo={docInfo} />; + } + return <LanguageMarker docInfo={docInfo} />; + } +}; + +const PathMarker = (props: { docInfo: DocumentInfo }) => { + return <Text>{getPathMarker(props.docInfo)}</Text>; +}; + +const LanguageMarker = (props: { docInfo: DocumentInfo }) => { + return <Text>{getLanguageMarker(props.docInfo)}</Text>; +}; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/recentEdits.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/recentEdits.tsx new file mode 100644 index 0000000..45b1b10 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/recentEdits.tsx @@ -0,0 +1,134 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { Chunk, ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { newLineEnded } from '../../../../prompt/src/languageMarker'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { + CompletionRequestData, + isCompletionRequestData, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { FullRecentEditsProvider, ICompletionsRecentEditsProviderService } from '../recentEdits/recentEditsProvider'; +import { RecentEdit } from '../recentEdits/recentEditsReducer'; + +export function editIsTooCloseToCursor( + edit: RecentEdit, + filterByCursorLine: boolean = false, + cursorLine: number | undefined = undefined, + activeDocDistanceLimitFromCursor: number | undefined +): boolean { + if (filterByCursorLine) { + if (cursorLine === undefined || activeDocDistanceLimitFromCursor === undefined) { + throw new Error( + 'cursorLine and activeDocDistanceLimitFromCursor are required when filterByCursorLine is true' + ); + } + } + + const startLineNumber = edit.startLine - 1; + const endLineNumber = edit.endLine - 1; + + if ( + filterByCursorLine && + (Math.abs(startLineNumber - cursorLine!) <= activeDocDistanceLimitFromCursor! || + Math.abs(endLineNumber - cursorLine!) <= activeDocDistanceLimitFromCursor!) + ) { + // skip over a diff that's too close to the cursor + // this isn't cached since the cursor moves + return true; + } + return false; +} + +type RecentEditsProps = { + tdms: ICompletionsTextDocumentManagerService; + recentEditsProvider: ICompletionsRecentEditsProviderService; +} & PromptElementProps; + +/** + * Render the most recent edits in the prompt. + * @param props + * @param context + * @returns a <Text> element containing recent edit summaries, or undefined if there are no recent edits + */ +export const RecentEdits = (props: RecentEditsProps, context: ComponentContext) => { + const [prompt, setPrompt] = context.useState<string | undefined>(); + + context.useData(isCompletionRequestData, async (request: CompletionRequestData) => { + if (!request.document) { return; } + + const recentEditProvider = props.recentEditsProvider; + + if (recentEditProvider.isEnabled()) { + recentEditProvider.start(); + } else { + return; + } + + const recentEditsConfig = (recentEditProvider as FullRecentEditsProvider).config; + const recentEdits = recentEditProvider.getRecentEdits(); + + const filesIncluded = new Set<string>(); + const tdm = props.tdms; + const editSummaries: string[] = []; + + // Walk backwards through the recent edits (most recent first) until we hit the max files or max edits, whichever comes first + for (let i = recentEdits.length - 1; i >= 0; i--) { + // if we've hit the max edits, stop + if (editSummaries.length >= recentEditsConfig.maxEdits) { break; } + + const edit = recentEdits[i]; + + // If the file is excluded, skip it + if (!(await tdm.getTextDocument({ uri: edit.file }))) { continue; } + + // If adding an edit from this file would exceed the max files, skip it + const isNewFile = !filesIncluded.has(edit.file); + const projectedFileCount = filesIncluded.size + (isNewFile ? 1 : 0); + if (projectedFileCount > recentEditsConfig.maxFiles) { break; } + + const filterByCursorLine = edit.file === request.document?.uri; + const activeDocCursorLine = filterByCursorLine ? request.position.line : undefined; + + // Check if the edit is too close to the cursor line, if applicable, in which case we skip it + const editTooClose = editIsTooCloseToCursor( + edit, + filterByCursorLine, + activeDocCursorLine, + recentEditsConfig.activeDocDistanceLimitFromCursor + ); + if (editTooClose) { + continue; + } + + const summarizedEdit = recentEditProvider.getEditSummary(edit); + if (summarizedEdit) { + filesIncluded.add(edit.file); + const relativePathOrUri = tdm.getRelativePath({ uri: edit.file }); + editSummaries.unshift(newLineEnded(`File: ${relativePathOrUri}`) + newLineEnded(summarizedEdit)); + } + } + + if (editSummaries.length === 0) { + setPrompt(undefined); + return; + } + + const newPrompt = + newLineEnded('These are recently edited files. Do not suggest code that has been deleted.') + + editSummaries.join('') + + newLineEnded('End of recent edits'); + + setPrompt(newPrompt); + }); + + return prompt ? ( + <Chunk> + <Text>{prompt}</Text> + </Chunk> + ) : undefined; +}; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/similarFiles.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/similarFiles.tsx new file mode 100644 index 0000000..0ed5a7e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/similarFiles.tsx @@ -0,0 +1,118 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { Chunk, ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { DocumentInfoWithOffset, PromptOptions } from '../../../../prompt/src/prompt'; +import { getSimilarSnippets } from '../../../../prompt/src/snippetInclusion/similarFiles'; +import { announceSnippet } from '../../../../prompt/src/snippetInclusion/snippets'; +import { getSimilarFilesOptions } from '../../experiments/similarFileOptionsProvider'; +import { TelemetryWithExp } from '../../telemetry'; +import { TextDocumentContents } from '../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { + CompletionRequestData, + CompletionRequestDocument, + isCompletionRequestData, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { getPromptOptions } from '../prompt'; +import { NeighborsCollection, NeighborSource } from '../similarFiles/neighborFiles'; + +type SimilarFilesProps = { + instantiationService: IInstantiationService; + tdms: ICompletionsTextDocumentManagerService; +} & PromptElementProps; + +type SimilarFileSnippet = { + headline: string; + snippet: string; + score: number; +}; + +export const SimilarFiles = (props: SimilarFilesProps, context: ComponentContext) => { + const [document, setDocument] = context.useState<CompletionRequestDocument>(); + const [similarFiles, setSimilarFiles] = context.useState<SimilarFileSnippet[]>([]); + + context.useData(isCompletionRequestData, async (requestData: CompletionRequestData) => { + if (requestData.document.uri !== document?.uri) { + setSimilarFiles([]); + } + setDocument(requestData.document); + + let files: { docs: NeighborsCollection } = NeighborSource.defaultEmptyResult(); + if (!requestData.turnOffSimilarFiles) { + files = await props.instantiationService.invokeFunction(async acc => await NeighborSource.getNeighborFilesAndTraits( + acc, + requestData.document.uri, + requestData.document.detectedLanguageId, + requestData.telemetryData, + requestData.cancellationToken, + requestData.data + )); + } + + const similarFiles = await produceSimilarFiles( + requestData.telemetryData, + requestData.document, + requestData, + files + ); + setSimilarFiles(similarFiles); + }); + + async function produceSimilarFiles( + telemetryData: TelemetryWithExp, + doc: TextDocumentContents, + requestData: CompletionRequestData, + files: { + docs: NeighborsCollection; + } + ): Promise<SimilarFileSnippet[]> { + const promptOptions = props.instantiationService.invokeFunction(getPromptOptions, telemetryData, doc.detectedLanguageId); + const similarSnippets = await findSimilarSnippets(promptOptions, telemetryData, doc, requestData, files); + return similarSnippets + .filter(s => s.snippet.length > 0) + .sort((a, b) => a.score - b.score) + .map(s => { + return { ...announceSnippet(s), score: s.score }; + }); + } + + async function findSimilarSnippets( + promptOptions: PromptOptions, + telemetryData: TelemetryWithExp, + doc: TextDocumentContents, + requestData: CompletionRequestData, + files: { docs: NeighborsCollection } + ) { + const similarFilesOptions = + promptOptions.similarFilesOptions || + props.instantiationService.invokeFunction(getSimilarFilesOptions, telemetryData, doc.detectedLanguageId); + const tdm = props.tdms; + const relativePath = tdm.getRelativePath(doc); + const docInfo: DocumentInfoWithOffset = { + uri: doc.uri, + source: doc.getText(), + offset: doc.offsetAt(requestData.position), + relativePath, + languageId: doc.detectedLanguageId, + }; + return await getSimilarSnippets(docInfo, Array.from(files.docs.values()), similarFilesOptions); + } + + return <>{...similarFiles.map((file, index) => <SimilarFile snippet={file} />)}</>; +}; + +// TODO: change Chunk for KeepTogether +const SimilarFile = (props: { snippet: SimilarFileSnippet }, context: ComponentContext) => { + return ( + <Chunk> + <Text>{props.snippet.headline}</Text> + <Text>{props.snippet.snippet}</Text> + </Chunk> + ); +}; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/splitContextPrompt.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/splitContextPrompt.tsx new file mode 100644 index 0000000..8f1b83b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/splitContextPrompt.tsx @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { ICompletionsRecentEditsProviderService } from '../recentEdits/recentEditsProvider'; +import { CodeSnippets } from './codeSnippets'; +import { AdditionalCompletionsContext, StableCompletionsContext } from './completionsContext'; +import { DocumentPrefix, DocumentSuffix } from './currentFile'; +import { Diagnostics } from './diagnostics'; +import { DocumentMarker } from './marker'; +import { RecentEdits } from './recentEdits'; +import { SimilarFiles } from './similarFiles'; +import { Traits } from './traits'; + +/** + * Function that returns the prompt structure for a code completion request following the split context prompt design + * that optimizes for cache hits. + */ +export function splitContextCompletionsPrompt(accessor: ServicesAccessor) { + const instantiationService = accessor.get(IInstantiationService); + const tdms = accessor.get(ICompletionsTextDocumentManagerService); + const recentEditsProvider = accessor.get(ICompletionsRecentEditsProviderService); + return ( + <> + <StableCompletionsContext> + <DocumentMarker tdms={tdms} weight={0.7} /> + <Traits weight={0.6} /> + <Diagnostics tdms={tdms} weight={0.65} /> + <CodeSnippets tdms={tdms} weight={0.9} /> + <SimilarFiles tdms={tdms} instantiationService={instantiationService} weight={0.8} /> + </StableCompletionsContext> + <DocumentSuffix weight={1} /> + <AdditionalCompletionsContext> + <RecentEdits tdms={tdms} recentEditsProvider={recentEditsProvider} weight={0.99} /> + </AdditionalCompletionsContext> + <DocumentPrefix weight={1} /> + </> + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/splitContextPromptRenderer.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/splitContextPromptRenderer.tsx new file mode 100644 index 0000000..9b8bef6 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/splitContextPromptRenderer.tsx @@ -0,0 +1,134 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { ComponentStatistics, PromptSnapshotNode } from '../../../../prompt/src/components/components'; +import { SnapshotWalker, WalkContextTransformer } from '../../../../prompt/src/components/walker'; +import { isContextNode } from './completionsContext'; +import { + CompletionsPromptRenderer, + normalizeLineEndings, + transformers, +} from './completionsPromptRenderer'; +import { BeforeCursor } from './currentFile'; +import { ElidedBlock, makeContextPrompt, makePrefixPrompt, WeightedBlock } from './elision'; + +let contextIndex = 0; +function resetContextIndex() { + contextIndex = 0; +} + +function getNextContextIndex() { + return contextIndex++; +} + +export class SplitContextPromptRenderer extends CompletionsPromptRenderer { + protected override formatPrefix: (elidedBlocks: ElidedBlock[]) => string = makePrefixPrompt; + protected override formatContext: ((elidedBlocks: ElidedBlock[]) => string[]) | undefined = makeContextPrompt; + + override processSnapshot( + snapshot: PromptSnapshotNode, + delimiter: string + ): { + prefixBlocks: WeightedBlock[]; + suffixBlock: WeightedBlock; + componentStatistics: ComponentStatistics[]; + } { + const prefixBlocks: WeightedBlock[] = []; + const suffixBlocks: WeightedBlock[] = []; + const componentStatistics: ComponentStatistics[] = []; + + // Store the status of the required prefix node + let foundPrefix = false; + + resetContextIndex(); + const walker = new SnapshotWalker(snapshot, splitContextTransformers); + walker.walkSnapshot((node, _parent, context) => { + if (node === snapshot) { + return true; + } + + if (node.statistics.updateDataTimeMs && node.statistics.updateDataTimeMs > 0) { + componentStatistics.push({ + componentPath: node.path, + updateDataTimeMs: node.statistics.updateDataTimeMs, + }); + } + + // Check for the presence of required prefix node + if (node.name === BeforeCursor.name) { + foundPrefix = true; + } + + if (node.value === undefined || node.value === '') { + // No need to process this node as it only adds whitespace + return true; + } + + const chunks = context.chunks as Set<string> | undefined; + const type = context.type as string | undefined; + if (type === 'suffix') { + // Suffix handling: Mark the child node with content as suffix + suffixBlocks.push({ + value: normalizeLineEndings(node.value), + type: 'suffix', + weight: context.weight as number, + componentPath: node.path, + nodeStatistics: node.statistics, + chunks, + source: context.source, + }); + } else { + const isPrefix = type === 'prefix'; + + // Add delimiter to non-prefix nodes + const nodeValueWithDelimiter = + isPrefix || node.value.endsWith(delimiter) ? node.value : node.value + delimiter; + prefixBlocks.push({ + type: isPrefix ? 'prefix' : 'context', + value: normalizeLineEndings(nodeValueWithDelimiter), + weight: context.weight as number, + componentPath: node.path, + nodeStatistics: node.statistics, + chunks, + source: context.source, + index: isPrefix ? undefined : (context.index as number), // index only set for context nodes + }); + } + return true; + }); + + if (!foundPrefix) { + throw new Error(`Node of type ${BeforeCursor.name} not found`); + } + if (suffixBlocks.length > 1) { + throw new Error(`Only one suffix is allowed`); + } + + const suffixBlock: WeightedBlock = + suffixBlocks.length === 1 + ? suffixBlocks[0] + : { + componentPath: '', + value: '', + weight: 1, + nodeStatistics: {}, + type: 'suffix', + }; + + return { prefixBlocks, suffixBlock, componentStatistics }; + } +} + +const splitContextTransformers: WalkContextTransformer[] = [ + ...transformers, + (node, _, context) => { + if (isContextNode(node)) { + return { ...context, index: getNextContextIndex() }; + } + return context; + }, +]; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/codeSnippets.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/codeSnippets.test.tsx new file mode 100644 index 0000000..c24641b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/codeSnippets.test.tsx @@ -0,0 +1,296 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import { CompletionRequestData } from '../../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { CodeSnippetWithId } from '../../contextProviders/contextItemSchemas'; +import { CodeSnippets } from '../codeSnippets'; + +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { PromptSnapshotNode } from '../../../../../prompt/src/components/components'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { extractNodesWitPath } from '../../../../../prompt/src/test/components/testHelpers'; +import { TelemetryWithExp } from '../../../telemetry'; +import { createLibTestingContext } from '../../../test/context'; +import { querySnapshot } from '../../../test/snapshot'; +import { createTextDocument, TestTextDocumentManager } from '../../../test/textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; + +suite('Code Snippets Component', function () { + let accessor: ServicesAccessor; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + }); + + test('Renders nothing if there are no code snippets', async function () { + try { + const snapshot = await renderCodeSnippets(accessor); + querySnapshot(snapshot.snapshot!, 'CodeSnippets'); + } catch (e) { + assert.ok((e as Error).message.startsWith('No children found at path segment ')); + } + }); + + test('Renders nothing if the code snippets array is empty', async function () { + try { + const snapshot = await renderCodeSnippets(accessor, []); + querySnapshot(snapshot.snapshot!, 'CodeSnippets'); + } catch (e) { + assert.ok((e as Error).message.startsWith('No children found at path segment ')); + } + }); + + test('Renders a single code snippet', async function () { + const codeSnippets: CodeSnippetWithId[] = [ + { + uri: 'file:///path/something.ts', + value: dedent` + function foo() { + return 1; + } + `, + id: '1', + type: 'CodeSnippet', + }, + ]; + + const snapshot = await renderCodeSnippets(accessor, codeSnippets); + + const chunks = querySnapshot(snapshot.snapshot!, 'CodeSnippets[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(chunks.length, 1); + const chunk = querySnapshot(snapshot.snapshot!, 'CodeSnippets[0].Chunk[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(chunk.length, 2); + assert.deepStrictEqual(chunk[1].props?.key, '1'); + assert.deepStrictEqual(chunk[1].props?.source, codeSnippets[0]); + // Assert content + assert.deepStrictEqual( + querySnapshot(snapshot.snapshot!, 'CodeSnippets[0].Chunk[0].Text'), + 'Compare this snippet from something.ts:' + ); + assert.deepStrictEqual( + querySnapshot(snapshot.snapshot!, 'CodeSnippets[0].Chunk["1"].Text'), + 'function foo() {\n\treturn 1;\n}' + ); + }); + + test('Renders snippet from subfolder', async function () { + const codeSnippets: CodeSnippetWithId[] = [ + { + uri: 'file:///c%3A/root/same.ts', + value: dedent` + function bar() { + return 1; + } + `, + id: '1', + type: 'CodeSnippet', + }, + { + uri: 'file:///c%3A/root/subfolder/something.ts', + value: dedent` + function foo() { + return 1; + } + `, + id: '2', + type: 'CodeSnippet', + }, + ]; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///c:/root' }]); + + const snapshot = await renderCodeSnippets(accessor, codeSnippets); + const chunks = querySnapshot(snapshot.snapshot!, 'CodeSnippets[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(chunks.length, 2); + + const firstChunk = querySnapshot(snapshot.snapshot!, 'CodeSnippets[0].Chunk[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(firstChunk.length, 2); + assert.deepStrictEqual(firstChunk[0].children?.[0].value, 'Compare this snippet from subfolder/something.ts:'); + assert.deepStrictEqual(firstChunk[1].props?.key, '2'); + assert.deepStrictEqual(firstChunk[1].props?.source, codeSnippets[1]); + + const secondChunk = querySnapshot(snapshot.snapshot!, 'CodeSnippets[1].Chunk[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(secondChunk.length, 2); + assert.deepStrictEqual(secondChunk[0].children?.[0].value, 'Compare this snippet from same.ts:'); + assert.deepStrictEqual(secondChunk[1].props?.key, '1'); + assert.deepStrictEqual(secondChunk[1].props?.source, codeSnippets[0]); + }); + + test('Renders multiple code snippets', async function () { + const codeSnippets: CodeSnippetWithId[] = [ + { + uri: 'file:///something.ts', + value: dedent` + function foo() { + return 1; + } + `, + id: '1', + type: 'CodeSnippet', + }, + { + uri: 'file:///somethingElse.ts', + value: dedent` + function bar() { + return 'two'; + } + `, + id: '2', + type: 'CodeSnippet', + }, + ]; + + const snapshot = await renderCodeSnippets(accessor, codeSnippets); + + const snippets = querySnapshot(snapshot.snapshot!, 'CodeSnippets[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(snippets.length, 2); + + const firstChunk = querySnapshot(snapshot.snapshot!, 'CodeSnippets[0].Chunk[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(firstChunk[0].children?.[0].value, 'Compare this snippet from somethingElse.ts:'); + assert.deepStrictEqual(firstChunk[1].props?.key, '2'); + assert.deepStrictEqual(firstChunk[1].props?.source, codeSnippets[1]); + + const secondChunk = querySnapshot(snapshot.snapshot!, 'CodeSnippets[1].Chunk[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(secondChunk[0].children?.[0].value, 'Compare this snippet from something.ts:'); + assert.deepStrictEqual(secondChunk[1].props?.key, '1'); + assert.deepStrictEqual(secondChunk[1].props?.source, codeSnippets[0]); + }); + + test('Merges together snippets with the same URI', async function () { + const codeSnippets: CodeSnippetWithId[] = [ + { + uri: 'file:///something.ts', + value: dedent` + function foo() { + return 1; + } + `, + id: '1', + type: 'CodeSnippet', + }, + { + uri: 'file:///something.ts', + value: dedent` + function bar() { + return 'two'; + } + `, + id: '2', + type: 'CodeSnippet', + }, + ]; + + const snapshot = await renderCodeSnippets(accessor, codeSnippets); + const result = querySnapshot(snapshot.snapshot!, 'CodeSnippets[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(result.length, 1); + + const chunk = querySnapshot(snapshot.snapshot!, 'CodeSnippets[0].Chunk[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(chunk.length, 4); + assert.deepStrictEqual(chunk[0].children?.[0].value, 'Compare these snippets from something.ts:'); + assert.deepStrictEqual(chunk[1].props?.key, '1'); + assert.deepStrictEqual(chunk[1].props?.source, codeSnippets[0]); + assert.deepStrictEqual(chunk[2].children?.[0].value, '---'); + assert.deepStrictEqual(chunk[3].props?.key, '2'); + assert.deepStrictEqual(chunk[3].props?.source, codeSnippets[1]); + }); + + test('Sorts snippets by ascending score of importance', async function () { + const codeSnippets: CodeSnippetWithId[] = [ + { + uri: 'file:///something.ts', + value: dedent` + function foo() { + return 1; + } + `, + importance: 10, + id: '1', + type: 'CodeSnippet', + }, + { + uri: 'file:///something.ts', + value: dedent` + function bar() { + return 'two'; + } + `, + importance: 5, + id: '2', + type: 'CodeSnippet', + }, + { + uri: 'file:///somethingElse.ts', + value: dedent` + function baz() { + return 'three'; + } + `, + importance: 7, + id: '3', + type: 'CodeSnippet', + }, + ]; + + const snapshot = await renderCodeSnippets(accessor, codeSnippets); + + const result = querySnapshot(snapshot.snapshot!, 'CodeSnippets[*]') as PromptSnapshotNode[]; + assert.deepStrictEqual(result.length, 2); + + assert.deepStrictEqual(extractNodesWitPath(snapshot.snapshot!), [ + '$[0].CodeSnippets', + '$[0].CodeSnippets[0].Chunk', + '$[0].CodeSnippets[0].Chunk[0].Text', + '$[0].CodeSnippets[0].Chunk[0].Text[0]', + '$[0].CodeSnippets[0].Chunk["3"].Text', + '$[0].CodeSnippets[0].Chunk["3"].Text[0]', + '$[0].CodeSnippets[1].Chunk', + '$[0].CodeSnippets[1].Chunk[0].Text', + '$[0].CodeSnippets[1].Chunk[0].Text[0]', + '$[0].CodeSnippets[1].Chunk["1"].Text', + '$[0].CodeSnippets[1].Chunk["1"].Text[0]', + '$[0].CodeSnippets[1].Chunk[2].Text', + '$[0].CodeSnippets[1].Chunk[2].Text[0]', + '$[0].CodeSnippets[1].Chunk["2"].Text', + '$[0].CodeSnippets[1].Chunk["2"].Text[0]', + ]); + }); +}); + +async function renderCodeSnippets(accessor: ServicesAccessor, codeSnippets?: CodeSnippetWithId[]) { + const document = createTextDocument( + 'file:///path/foo.ts', + 'typescript', + 0, + dedent` + const a = 1; + function f| + const b = 2; + ` + ); + const position = document.positionAt(document.getText().indexOf('|')); + + const tdms = accessor.get(ICompletionsTextDocumentManagerService); + const virtualPrompt = new VirtualPrompt(<CodeSnippets tdms={tdms} />); + const pipe = virtualPrompt.createPipe(); + + const completionRequestData: CompletionRequestData = { + document, + position, + telemetryData: TelemetryWithExp.createEmptyConfigForTesting(), + cancellationToken: new CancellationTokenSource().token, + maxPromptTokens: 1000, + data: undefined, + codeSnippets, + }; + + await pipe.pump(completionRequestData); + return virtualPrompt.snapshot(); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/completionsPromptRenderer.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/completionsPromptRenderer.test.tsx new file mode 100644 index 0000000..eb197d4 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/completionsPromptRenderer.test.tsx @@ -0,0 +1,994 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import { CancellationTokenSource, Position } from 'vscode-languageserver-protocol'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { Chunk, PromptElementProps, PromptSnapshotNode, Text } from '../../../../../prompt/src/components/components'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { TokenizerName } from '../../../../../prompt/src/tokenization'; +import { createCompletionRequestData } from '../../../test/completionsPrompt'; +import { createLibTestingContext } from '../../../test/context'; +import { createTextDocument } from '../../../test/textDocument'; +import { CodeSnippetWithId, TraitWithId } from '../../contextProviders/contextItemSchemas'; +import { CompletionsContext, StableCompletionsContext } from '../completionsContext'; +import { + CompletionsPromptRenderer, + CompletionsPromptRenderOptions, +} from '../completionsPromptRenderer'; +import { CurrentFile } from '../currentFile'; + +const MyNestedComponent = () => { + return ( + <> + <Text weight={0.5}>This goes first</Text> + <Text weight={0.6}>This goes last</Text> + </> + ); +}; + +const AnotherComponent = (props: PromptElementProps & { number: number }) => { + return <Text>This is a number {props.number ?? 0}</Text>; +}; + +const renderingOptions: CompletionsPromptRenderOptions = { + promptTokenLimit: 70, + suffixPercent: 20, + delimiter: '\n', + tokenizer: TokenizerName.o200k, + languageId: 'typescript', +}; + +const fullExpectedPrefix = + '// This is a number 1\n// This goes first\n// This goes last\n// This is a number 2\n// Raw text\n// Another raw text\nconst a = 1;\nfunction f'; +const fullExpectedSuffix = 'const b = 2;\nconst c = 3;'; + +for (const lineEnding of ['\n', '\r\n']) { + const fileUri = 'file:///path/basename.ts'; + const source = `const a = 1;${lineEnding}function f|${lineEnding}const b = 2;${lineEnding}const c = 3;`; + const textDocument = createTextDocument(fileUri, 'typescript', 0, source); + const position: Position = textDocument.positionAt(textDocument.getText().indexOf('|')); + suite(`Completions Prompt Renderer (line ending: ${JSON.stringify(lineEnding)})`, function () { + let accessor: ServicesAccessor; + let renderer: CompletionsPromptRenderer; + let snapshot: PromptSnapshotNode | undefined; + + setup(async function () { + accessor = createLibTestingContext().createTestingAccessor(); + renderer = new CompletionsPromptRenderer(); + const vPrompt = new VirtualPrompt( + ( + <> + <CompletionsContext> + <AnotherComponent number={1} /> + <MyNestedComponent /> + {/* This is intentionally placed here so that it's far from the other AnotherComponent*/} + <AnotherComponent number={2} /> + <> + {/* This is intentionally in a fragment to check that it's skipped */} + <Text>Raw text</Text> + </> + <> + <Text>Another raw text</Text> + </> + </CompletionsContext> + <CurrentFile /> + </> + ) + ); + const pipe = vPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + ({ snapshot } = vPrompt.snapshot()); + }); + + test('renders prefix and suffix based on completions doc position', function () { + const prompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, fullExpectedPrefix); + assert.deepStrictEqual(prompt.prefixTokens, 43); + assert.deepStrictEqual(prompt.suffix, fullExpectedSuffix); + assert.deepStrictEqual(prompt.suffixTokens, 12); + assert.deepStrictEqual(prompt.context, undefined); + }); + + test('single context with comments', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.prefix, '// This is context\n'); + assert.deepStrictEqual(rendered.context, undefined); + }); + + test('multiple context with comments', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.prefix, '// This is context\n// This is more context\n'); + assert.deepStrictEqual(rendered.context, undefined); + }); + + test('multiple context blocks', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <CompletionsContext> + <Text>This is other context</Text> + <Text>This is extra context</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual( + rendered.prefix, + '// This is context\n// This is more context\n// This is other context\n// This is extra context\n' + ); + assert.deepStrictEqual(rendered.context, undefined); + }); + + test('multiple types of context blocks ', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <StableCompletionsContext> + <Text>This is other context</Text> + <Text>This is extra context</Text> + </StableCompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual( + rendered.prefix, + '// This is context\n// This is more context\n// This is other context\n// This is extra context\n' + ); + assert.deepStrictEqual(rendered.context, undefined); + }); + + test('renders prefix and suffix using configured delimiter', function () { + const expectedPrefix = + '// This is a number 1?// This goes first?// This goes last?// This is a number 2?// Raw text?// Another raw text?const a = 1;\nfunction f'; + + const prompt = renderer.render(snapshot!, { ...renderingOptions, delimiter: '?' }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, expectedPrefix); + assert.deepStrictEqual(prompt.suffix, fullExpectedSuffix); + assert.deepStrictEqual(prompt.prefixTokens, 43); + assert.deepStrictEqual(prompt.suffixTokens, 12); + }); + + test('renders delimiter only if components do not already end with delimiter', function () { + const expectedPrefix = + '// This is a number 1text// This goes firsttext// This goes lasttext// This is a number 2text// Raw text// Another raw textconst a = 1;\nfunction f'; + + const prompt = renderer.render(snapshot!, { ...renderingOptions, delimiter: 'text' }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, expectedPrefix); + assert.deepStrictEqual(prompt.suffix, fullExpectedSuffix); + assert.deepStrictEqual(prompt.prefixTokens, 41); + assert.deepStrictEqual(prompt.suffixTokens, 12); + }); + + test('uses configured tokenizer', function () { + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + tokenizer: TokenizerName.cl100k, + }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefixTokens, 43); + assert.deepStrictEqual(prompt.suffixTokens, 12); + }); + + test('computes metadata with stable updateDataTimeMs tolerance', function () { + const prompt1 = renderer.render(snapshot!, renderingOptions); + const prompt2 = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(prompt1.status, 'ok'); + assert.deepStrictEqual(prompt2.status, 'ok'); + + const metadata1 = prompt1.metadata; + const metadata2 = prompt2.metadata; + + assert.deepStrictEqual(metadata1.renderId, 0); + assert.deepStrictEqual(metadata2.renderId, 1); + assert.ok(metadata1.renderTimeMs > 0); + assert.ok(metadata1.elisionTimeMs > 0); + const expectedComponents = [ + { + componentPath: '$.f[1].CurrentFile', + }, + { + componentPath: '$.f[0].CompletionsContext[0].AnotherComponent[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[1].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[0].CompletionsContext[2].AnotherComponent[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[0].CompletionsContext[3].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[4].f[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 12, + actualTokens: 12, + }, + ]; + + expectedComponents.forEach(expected => { + const actual = metadata1.componentStatistics.find(s => s.componentPath === expected.componentPath); + assert.ok(actual, `Component ${expected.componentPath} not found`); + assert.strictEqual( + actual.expectedTokens, + expected.expectedTokens, + `Expected tokens for ${expected.componentPath} do not match` + ); + assert.strictEqual(actual.actualTokens, expected.actualTokens); + // Instead of a fixed number, just ensure updateDataTimeMs is a non-negative number. + if (actual.updateDataTimeMs) { + assert.ok( + typeof actual.updateDataTimeMs === 'number' && actual.updateDataTimeMs >= 0, + `Expected updateDataTimeMs for ${expected.componentPath} to be a non-negative number` + ); + } + }); + }); + + test('computes usage statistics ignoring updateDataTimeMs field', function () { + const rendered = renderer.render(snapshot!, renderingOptions); + assert.deepStrictEqual(rendered.status, 'ok'); + const metadata = rendered.metadata; + // Make updateDataTimeMs a constant value to ensure it doesn't affect the test. + const actualStatsFiltered = metadata.componentStatistics.map(stats => { + if (stats.updateDataTimeMs) { + stats.updateDataTimeMs = 42; + } + return stats; + }); + const expectedStatsFiltered = [ + { + componentPath: '$.f[1].CurrentFile', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[0].AnotherComponent[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[1].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[0].CompletionsContext[2].AnotherComponent[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[0].CompletionsContext[3].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[4].f[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 12, + actualTokens: 12, + }, + ]; + assert.deepStrictEqual(actualStatsFiltered, expectedStatsFiltered); + }); + + test('propagates source via statistics', function () { + const trait: TraitWithId = { + name: 'trait', + value: 'value', + id: 'traitid', + type: 'Trait', + }; + const codeSnippet: CodeSnippetWithId = { + uri: 'file://foo.ts', + value: 'value', + id: 'traitid', + type: 'CodeSnippet', + }; + const prompt = ( + <> + <CompletionsContext> + <Text source={trait}>This is a trait</Text> + <Chunk source={codeSnippet}> + <Text>This is a code snippet</Text> + </Chunk> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, renderingOptions); + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.ok(renderedPrompt.metadata.componentStatistics.find(s => s.source === trait)); + assert.ok(renderedPrompt.metadata.componentStatistics.find(s => s.source === codeSnippet)); + }); + + test('elides prefix', function () { + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 20, + suffixPercent: 0, + }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, '// Raw text\n// Another raw text\nconst a = 1;\nfunction f'); + assert.deepStrictEqual(prompt.suffix, ''); + }); + + test('elides suffix (from the end!)', function () { + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 30, + suffixPercent: 10, + }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.suffix, 'const b ='); + }); + + test('elides both prefix and suffix partially', function () { + // Use tighter token limits to force partial elision on both sides. + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 20, + suffixPercent: 10, + }); + // We don't have the exact expected strings, but we verify that both prefix and suffix + // have been elided compared to the full expectations. + assert.strictEqual(prompt.status, 'ok'); + // The elided prefix should be shorter than the full expected one. + assert.ok(prompt.prefix.length < fullExpectedPrefix.length, 'Expected prefix to be elided'); + // The elided suffix should also be shorter than the full expected suffix, if any elision took place. + if (fullExpectedSuffix.length > 0) { + assert.ok(prompt.suffix.length < fullExpectedSuffix.length, 'Expected suffix to be elided'); + } + }); + + test('generates prompt metadata', function () { + const rendered = renderer.render(snapshot!, renderingOptions); + assert.deepStrictEqual(rendered.status, 'ok'); + const metadata = rendered.metadata; + assert.ok(metadata.renderId === 0); + assert.ok(metadata.elisionTimeMs > 0); + assert.ok(metadata.renderTimeMs > 0); + assert.ok(metadata.updateDataTimeMs > 0); + assert.deepStrictEqual(metadata.tokenizer, TokenizerName.o200k); + }); + + test('computes usage statistics after elision', function () { + const rendered = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 40, + suffixPercent: 10, + }); + assert.deepStrictEqual(rendered.status, 'ok'); + const metadata = rendered.metadata; + const actualStatsFiltered = metadata.componentStatistics.map(stats => { + if (stats.updateDataTimeMs) { + stats.updateDataTimeMs = 42; + } + return stats; + }); + assert.deepStrictEqual( + actualStatsFiltered.reduce((acc, curr) => acc + (curr.actualTokens ?? 0), 0), + 34 + ); + assert.deepStrictEqual(actualStatsFiltered, [ + { + componentPath: '$.f[1].CurrentFile', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[0].AnotherComponent[0].Text[0]', + expectedTokens: 8, + actualTokens: 0, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[1].Text[0]', + expectedTokens: 5, + actualTokens: 0, + }, + { + componentPath: '$.f[0].CompletionsContext[2].AnotherComponent[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[0].CompletionsContext[3].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[4].f[0].Text[0]', + expectedTokens: 5, + actualTokens: 5, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 12, + actualTokens: 4, + }, + ]); + }); + + function createStringWithNLines(n: number, baseText: string): string { + let result = ''; + for (let i = 1; i <= n; i++) { + result += `${baseText}${i}\n`; + } + return result; + } + + test('uses cached suffix if similar enough', async function () { + const firstSuffix = createStringWithNLines(15, 'a') + createStringWithNLines(10, 'b'); + const secondSuffix = createStringWithNLines(15, 'a') + createStringWithNLines(10, 'c'); + const renderOptionsWithSuffix: CompletionsPromptRenderOptions = { + ...renderingOptions, + promptTokenLimit: 205, + suffixPercent: 50, + }; + const textDocumentWithFirstSuffix = createTextDocument( + fileUri, + 'typescript', + 0, + 'function f|\n' + firstSuffix + ); + const position = textDocumentWithFirstSuffix.positionAt(textDocumentWithFirstSuffix.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithFirstSuffix, position)); + // Snapshot caches the suffix + virtualPrompt.snapshot(); + + // The position is the same, since the start of the document doesn't change + const textDocumentWithSecondSuffix = createTextDocument( + fileUri, + 'typescript', + 1, + 'function f|\n' + secondSuffix + ); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithSecondSuffix, position)); + const { snapshot: snapshotWithDefaultThreshold } = virtualPrompt.snapshot(); + + // the first suffix is used, since they are similar enough + const renderedWithDefaultThreshold = renderer.render( + snapshotWithDefaultThreshold!, + renderOptionsWithSuffix + ); + assert.deepStrictEqual(renderedWithDefaultThreshold.status, 'ok'); + assert.deepStrictEqual(renderedWithDefaultThreshold.suffix, firstSuffix); + + await pipe.pump( + createCompletionRequestData( + accessor, + textDocumentWithSecondSuffix, + position, + undefined, + undefined, + undefined, + 3 + ) + ); + const { snapshot: snapshotWithLowerThreshold } = virtualPrompt.snapshot(); + + // The second suffix is used, since the matching threshold is lower + const renderedWithLowerThreshold = renderer.render(snapshotWithLowerThreshold!, renderOptionsWithSuffix); + assert.deepStrictEqual(renderedWithLowerThreshold.status, 'ok'); + assert.deepStrictEqual(renderedWithLowerThreshold.suffix, secondSuffix); + }); + + test('does not use cached suffix if not similar enough', async function () { + const firstSuffix = createStringWithNLines(15, 'a') + createStringWithNLines(10, 'b'); + const secondSuffix = createStringWithNLines(3, 'a') + createStringWithNLines(22, 'c'); + const renderOptionsWithSuffix: CompletionsPromptRenderOptions = { + ...renderingOptions, + promptTokenLimit: 205, + suffixPercent: 50, + }; + const textDocumentWithFirstSuffix = createTextDocument( + fileUri, + 'typescript', + 0, + 'function f|\n' + firstSuffix + ); + const position = textDocumentWithFirstSuffix.positionAt(textDocumentWithFirstSuffix.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithFirstSuffix, position)); + // Snapshot caches the suffix + virtualPrompt.snapshot(); + + // The position is the same, since the start of the document doesn't change + const textDocumentWithSecondSuffix = createTextDocument( + fileUri, + 'typescript', + 1, + 'function f|\n' + secondSuffix + ); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithSecondSuffix, position)); + const { snapshot } = virtualPrompt.snapshot(); + + // the second suffix is used, since they are not similar enough + const rendered = renderer.render(snapshot!, renderOptionsWithSuffix); + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.suffix, secondSuffix); + }); + + test('suffix can be empty', async function () { + const textDocumentWithoutSuffix = createTextDocument(fileUri, 'typescript', 0, 'function f|'); + const position = textDocumentWithoutSuffix.positionAt(textDocumentWithoutSuffix.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithoutSuffix, position)); + const { snapshot } = virtualPrompt.snapshot(); + const promptWithoutSuffix = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(promptWithoutSuffix.status, 'ok'); + assert.deepStrictEqual(promptWithoutSuffix.suffix, ''); + assert.deepStrictEqual(promptWithoutSuffix.prefix, 'function f'); + }); + + test('prefix can be empty', async function () { + const emptyTextDocument = createTextDocument(fileUri, 'typescript', 0, '|\nconst b = 2;'); + const position = emptyTextDocument.positionAt(emptyTextDocument.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, emptyTextDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const emptyPrompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(emptyPrompt.status, 'ok'); + assert.deepStrictEqual(emptyPrompt.prefix, ''); + assert.deepStrictEqual(emptyPrompt.suffix, 'const b = 2;'); + }); + + test('prefix and suffix can be empty', async function () { + const emptyTextDocument = createTextDocument(fileUri, 'typescript', 0, ''); + const position = emptyTextDocument.positionAt(0); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, emptyTextDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const emptyPrompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(emptyPrompt.status, 'ok'); + assert.deepStrictEqual(emptyPrompt.prefix, ''); + assert.deepStrictEqual(emptyPrompt.suffix, ''); + }); + + test('cancels rendering when token has been cancelled', function () { + const cts = new CancellationTokenSource(); + + cts.cancel(); + const prompt = renderer.render(snapshot!, renderingOptions, cts.token); + + assert.deepStrictEqual(prompt.status, 'cancelled'); + }); + + test('throws error when tree does not contain completions document component', function () { + const promptCompletionsDocument = ( + <> + <Text>Whatever</Text> + </> + ); + + const virtualPrompt = new VirtualPrompt(promptCompletionsDocument); + const { snapshot } = virtualPrompt.snapshot(); + const prompt = renderer.render(snapshot!, renderingOptions); + + assert.strictEqual(prompt.status, 'error'); + assert.strictEqual(prompt.error.message, `Node of type ${CurrentFile.name} not found`); + }); + + test('renders empty prefix and suffix if no data is sent', function () { + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const emptyPrompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(emptyPrompt.status, 'ok'); + assert.deepStrictEqual(emptyPrompt.prefix, ''); + assert.deepStrictEqual(emptyPrompt.suffix, ''); + }); + + test('does not re-render if no data matching the expected structure is sent', async function () { + const textDocument = createTextDocument( + fileUri, + 'typescript', + 0, + `import * from './foo.ts'\n|\nfunction f` + ); + const position = textDocument.positionAt(textDocument.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + + // First render + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const renderedPrompt = renderer.render(snapshot!, renderingOptions); + + // Second render + const { snapshot: snapshotTwo } = virtualPrompt.snapshot(); + const renderedPromptTwo = renderer.render(snapshotTwo!, renderingOptions); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPromptTwo.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, `import * from './foo.ts'\n`); + assert.deepStrictEqual(renderedPrompt.prefix, renderedPromptTwo.prefix); + assert.deepStrictEqual(renderedPrompt.suffix, 'function f'); + assert.deepStrictEqual(renderedPrompt.suffix, renderedPromptTwo.suffix); + }); + + test('re-renders if new data matching the expected structure is sent', async function () { + const textDocument = createTextDocument( + fileUri, + 'typescript', + 0, + `import * from './foo.ts'\n|\nfunction f` + ); + const position = textDocument.positionAt(textDocument.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + + // First render + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const renderedPrompt = renderer.render(snapshot!, renderingOptions); + + // Second render + const updatedTextDocument = createTextDocument( + fileUri, + 'typescript', + 1, // Notice version change + `import * from './bar.ts'\n|\nfunction g` + ); + const updatedPosition = updatedTextDocument.positionAt(updatedTextDocument.getText().indexOf('|')); + + await pipe.pump(createCompletionRequestData(accessor, updatedTextDocument, updatedPosition)); + const { snapshot: snapshotTwo } = virtualPrompt.snapshot(); + const renderedPromptTwo = renderer.render(snapshotTwo!, renderingOptions); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPromptTwo.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, `import * from './foo.ts'\n`); + assert.deepStrictEqual(renderedPromptTwo.prefix, `import * from './bar.ts'\n`); + assert.deepStrictEqual(renderedPrompt.suffix, 'function f'); + assert.deepStrictEqual(renderedPromptTwo.suffix, 'function g'); + }); + + test('Elides Chunk completely', function () { + const prompt = ( + <> + <CompletionsContext> + <Chunk weight={0.5}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text>Outside Text</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 10, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, '// Outside Text\n'); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('Elides Chunk completely while respecting lower weights', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.5}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 16, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, '// Outside Text 1\n// Outside Text 2\n'); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('Elides Chunk completely in case of exceeding the limit even with higher weight', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.5}>Outside Text 1</Text> + <Chunk weight={0.7}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.8}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 14, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, '// Outside Text 1\n// Outside Text 2\n'); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('Prefers higher weighted Chunk over lower weighted separate components', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.8}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 14, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, '// Chunk Text 1\n// Chunk Text 2\n'); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('If a nested chunk is elided first, the outer chunks is kept', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.5}> + <Text>Chunk Text 1</Text> + <Chunk weight={0.5}> + <Text>Nested Chunk Text 1</Text> + <Text>Nested Chunk Text 2</Text> + </Chunk> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 35, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual( + renderedPrompt.prefix, + '// Outside Text 1\n// Chunk Text 1\n// Chunk Text 2\n// Outside Text 2\n' + ); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('If the outer chunk is elided first, the inner chunk is also elided', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.5}> + <Text weight={0.5}>Chunk Text 1</Text> + <Chunk> + <Text>Nested Chunk Text 1</Text> + <Text>Nested Chunk Text 2</Text> + </Chunk> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 37, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, '// Outside Text 1\n// Outside Text 2\n'); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + }); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/contextProviderBridge.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/components/test/contextProviderBridge.test.ts new file mode 100644 index 0000000..2daeb0b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/contextProviderBridge.test.ts @@ -0,0 +1,136 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CodeSnippet, ContextProvider, ContextResolver, SupportedContextItem, Trait, type DiagnosticBag } from '../../../../../types/src'; +import { createCompletionState } from '../../../completionState'; +import { ICompletionsFeaturesService } from '../../../experiments/featuresService'; +import { TelemetryWithExp } from '../../../telemetry'; +import { createLibTestingContext } from '../../../test/context'; +import { createTextDocument } from '../../../test/textDocument'; +import { LocationFactory } from '../../../textDocument'; +import { ICompletionsContextProviderRegistryService } from '../../contextProviderRegistry'; +import { ContextProviderBridge } from './../contextProviderBridge'; + +suite('Context Provider Bridge', function () { + let accessor: ServicesAccessor; + let bridge: ContextProviderBridge; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + const featuresService = accessor.get(ICompletionsFeaturesService); + accessor.get(ICompletionsContextProviderRegistryService).registerContextProvider(new TestContextProvider()); + featuresService.contextProviders = () => ['testContextProvider']; + bridge = accessor.get(IInstantiationService).createInstance(ContextProviderBridge); + }); + + test('await context resolution by id', async function () { + const state = testCompletionState(); + + bridge.schedule(state, 'id', 'opId', TelemetryWithExp.createEmptyConfigForTesting()); + const items = await bridge.resolution('id'); + + assert.deepStrictEqual(items.length, 1); + assert.deepStrictEqual(items[0].providerId, 'testContextProvider'); + assert.deepStrictEqual((items[0].data[0] as Trait).name, 'test'); + assert.deepStrictEqual((items[0].data[0] as Trait).value, 'test'); + }); + + test('await context resolution by id twice', async function () { + const state = testCompletionState(); + bridge.schedule(state, 'id', 'opId', TelemetryWithExp.createEmptyConfigForTesting()); + + const items1 = await bridge.resolution('id'); + const items2 = await bridge.resolution('id'); + + assert.deepStrictEqual(items1.length, 1); + assert.deepStrictEqual(items1[0].providerId, 'testContextProvider'); + assert.deepStrictEqual((items1[0].data[0] as Trait).name, 'test'); + assert.deepStrictEqual((items1[0].data[0] as Trait).value, 'test'); + assert.deepStrictEqual(items1, items2); + }); + + test('no schedule called returns empty array', async function () { + const items = await bridge.resolution('unknown-id'); + + assert.deepStrictEqual(items, []); + }); + + test('error in context resolution', async function () { + const featuresService = accessor.get(ICompletionsFeaturesService); + accessor.get(ICompletionsContextProviderRegistryService).registerContextProvider( + new TestContextProvider({ shouldThrow: true, id: 'errorProvider' }) + ); + featuresService.contextProviders = () => ['errorProvider']; + const errorBridge = accessor.get(IInstantiationService).createInstance(ContextProviderBridge); + const state = testCompletionState(); + + errorBridge.schedule(state, 'err-id', 'opId', TelemetryWithExp.createEmptyConfigForTesting()); + const items = await errorBridge.resolution('err-id'); + + const errorItem = items.find(i => i.providerId === 'errorProvider'); + assert.deepStrictEqual(errorItem?.resolution, 'error'); + }); + + test('multiple schedules and resolutions', async function () { + const state1 = testCompletionState(); + const state2 = testCompletionState(); + + bridge.schedule(state1, 'id1', 'opId', TelemetryWithExp.createEmptyConfigForTesting()); + bridge.schedule(state2, 'id2', 'opId', TelemetryWithExp.createEmptyConfigForTesting()); + + const items1 = await bridge.resolution('id1'); + const items2 = await bridge.resolution('id2'); + + assert.deepStrictEqual(items1.length, 1); + assert.deepStrictEqual(items2.length, 1); + }); + + test('empty provider list returns empty array', async function () { + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.contextProviders = () => []; + const instantiationService = createLibTestingContext().createTestingAccessor().get(IInstantiationService); + bridge = instantiationService.createInstance(ContextProviderBridge); + const state = testCompletionState(); + + bridge.schedule(state, 'empty-id', 'opId', TelemetryWithExp.createEmptyConfigForTesting()); + const items = await bridge.resolution('empty-id'); + + assert.deepStrictEqual(items, []); + }); + + function testCompletionState() { + const doc = createTextDocument('file:///fizzbuzz.go', 'go', 1, 'code'); + const position = LocationFactory.position(3, 0); + return createCompletionState(doc, position); + } +}); + +class TestContextResolver implements ContextResolver<SupportedContextItem> { + private shouldThrow: boolean; + constructor(opts?: { shouldThrow?: boolean }) { + this.shouldThrow = opts?.shouldThrow ?? false; + } + + async *resolve(): AsyncIterable<SupportedContextItem> { + if (this.shouldThrow) { + throw new Error('Test error'); + } + yield Promise.resolve({ name: 'test', value: 'test' }); + } +} + +class TestContextProvider implements ContextProvider<Trait | CodeSnippet | DiagnosticBag> { + id: string; + selector: string[]; + resolver: ContextResolver<CodeSnippet | Trait | DiagnosticBag>; + + constructor(opts?: { shouldThrow?: boolean; id?: string }) { + this.id = opts?.id ?? 'testContextProvider'; + this.selector = ['*']; + this.resolver = new TestContextResolver({ shouldThrow: opts?.shouldThrow }); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/currentFile.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/currentFile.test.tsx new file mode 100644 index 0000000..785a27e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/currentFile.test.tsx @@ -0,0 +1,82 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { CurrentFile } from '../../../prompt/components/currentFile'; +import { createCompletionRequestData } from '../../../test/completionsPrompt'; +import { createLibTestingContext } from '../../../test/context'; +import { querySnapshot } from '../../../test/snapshot'; +import { createTextDocument } from '../../../test/textDocument'; + +suite('Completions Prompt Renderer', function () { + let accessor: ServicesAccessor; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + }); + + test('uses full before cursor if within limit', async function () { + const snapshot = await createSnapshot(1000); + + const value = querySnapshot(snapshot.snapshot!, 'CurrentFile[0].f[0].BeforeCursor[0].Text') as string; + assert.deepStrictEqual(value, 'const a = 1;\nfunction f'); + }); + + test('trims before cursor if exceeding limit', async function () { + const snapshot = await createSnapshot(2); + + const value = querySnapshot(snapshot.snapshot!, 'CurrentFile[0].f[0].BeforeCursor[0].Text') as string; + assert.deepStrictEqual(value, 'nction f'); + }); + + test('uses full after cursor if within limit', async function () { + const snapshot = await createSnapshot(1000); + + const value = querySnapshot(snapshot.snapshot!, 'CurrentFile[0].f[1].AfterCursor[0].Text') as string; + assert.deepStrictEqual(value, 'const b = 2;'); + }); + + test('trims after cursor if exceeding limit', async function () { + const snapshot = await createSnapshot(2); + + const value = querySnapshot(snapshot.snapshot!, 'CurrentFile[0].f[1].AfterCursor[0].Text') as string; + assert.deepStrictEqual(value, 'const '); + }); + + const createSnapshot = async (maxPromptTokens: number) => { + const textDocument = createTextDocument( + 'file:///path/basename', + 'typescript', + 0, + dedent` + const a = 1; + function f| + const b = 2; + ` + ); + const position = textDocument.positionAt(textDocument.getText().indexOf('|')); + const virtualPrompt = new VirtualPrompt(<CurrentFile />); + const pipe = virtualPrompt.createPipe(); + const data = createCompletionRequestData( + accessor, + textDocument, + position, + undefined, + undefined, + false, + undefined, + maxPromptTokens + ); + + await pipe.pump(data); + + return virtualPrompt.snapshot(); + }; +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/marker.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/marker.test.tsx new file mode 100644 index 0000000..1dfe8e1 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/marker.test.tsx @@ -0,0 +1,68 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { DocumentMarker } from '../../../prompt/components/marker'; +import { createCompletionRequestData } from '../../../test/completionsPrompt'; +import { createLibTestingContext } from '../../../test/context'; +import { querySnapshot } from '../../../test/snapshot'; +import { createTextDocument, InMemoryNotebookDocument, TestTextDocumentManager } from '../../../test/textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; + +suite('Document Marker', function () { + let accessor: ServicesAccessor; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + }); + + test('creates path with relative path', async function () { + const marker = await renderMarker(accessor, 'file:///path/basename'); + + assert.deepStrictEqual(marker, 'Path: basename'); + }); + + test('creates language marker with untitled document', async function () { + const marker = await renderMarker(accessor, 'untitled:uri'); + + assert.deepStrictEqual(marker, 'Language: typescript'); + }); + + test('creates language marker with relative path present but type is notebook', async function () { + const textDocument = createTextDocument('vscode-notebook:///mynotebook.ipynb', 'typescript', 0, ''); + (accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager).setNotebookDocument( + textDocument, + new InMemoryNotebookDocument([]) + ); + const marker = await renderMarker(accessor, textDocument.uri); + + assert.deepStrictEqual(marker, 'Language: typescript'); + }); + + async function renderMarker(accessor: ServicesAccessor, uri: string) { + const textDocument = createTextDocument( + uri, + 'typescript', + 0, + dedent` + const a = 1; + function f| + const b = 2; + ` + ); + const tdms = accessor.get(ICompletionsTextDocumentManagerService); + const position = textDocument.positionAt(textDocument.getText().indexOf('|')); + const virtualPrompt = new VirtualPrompt(<DocumentMarker tdms={tdms} />); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + const snapshot = virtualPrompt.snapshot(); + return querySnapshot(snapshot.snapshot!, 'DocumentMarker.*.Text'); + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/recentEdits.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/recentEdits.test.tsx new file mode 100644 index 0000000..8ad6f42 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/recentEdits.test.tsx @@ -0,0 +1,417 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import { IIgnoreService } from '../../../../../../../../platform/ignore/common/ignoreService'; +import { SyncDescriptor } from '../../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { ICompletionsObservableWorkspace } from '../../../completionsObservableWorkspace'; +import { createCompletionRequestData } from '../../../test/completionsPrompt'; +import { createLibTestingContext } from '../../../test/context'; +import { querySnapshot } from '../../../test/snapshot'; +import { MockIgnoreService } from '../../../test/testContentExclusion'; +import { TestTextDocumentManager } from '../../../test/textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; +import { CompletionRequestDocument } from '../../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { CompletionsMutableObservableWorkspace } from '../../completionsPromptFactory/test/completionsPromptFactory.test'; +import { FullRecentEditsProvider, ICompletionsRecentEditsProviderService } from '../../recentEdits/recentEditsProvider'; +import { DiffHunk, RecentEdit, summarizeEdit } from '../../recentEdits/recentEditsReducer'; +import { RecentEdits, editIsTooCloseToCursor } from '../recentEdits'; + +class MockRecentEditsProvider extends FullRecentEditsProvider { + override getRecentEdits = () => [] as RecentEdit[]; + + override getEditSummary(edit: RecentEdit): string | null { + return summarizeEdit(edit, this.config); + } +} + +suite('Recent Edits Component', function () { + let accessor: ServicesAccessor; + let mockRecentEditsProvider: MockRecentEditsProvider; + let ignoreService: MockIgnoreService; + + setup(function () { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsObservableWorkspace, new CompletionsMutableObservableWorkspace()); + serviceCollection.define(ICompletionsRecentEditsProviderService, new SyncDescriptor(MockRecentEditsProvider, [undefined])); + serviceCollection.define(IIgnoreService, new MockIgnoreService()); + accessor = serviceCollection.createTestingAccessor(); + + ignoreService = accessor.get(IIgnoreService) as MockIgnoreService; + mockRecentEditsProvider = accessor.get(ICompletionsRecentEditsProviderService) as MockRecentEditsProvider; + }); + + test('renders nothing when recent edits are disabled', async function () { + mockRecentEditsProvider.isEnabled = () => false; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + const doc = tdm.setTextDocument('file:///foo.ts', 'typescript', 'const x = |;'); + + const snapshot = await createSnapshot(accessor, doc, '|'); + assert.throws(() => querySnapshot(snapshot, 'RecentEdits')); + }); + + test('renders recent edits correctly', async function () { + mockRecentEditsProvider.config.maxEdits = 5; + mockRecentEditsProvider.config.diffContextLines = 1; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = -1; + mockRecentEditsProvider.config.summarizationFormat = 'diff'; + mockRecentEditsProvider.config.maxFiles = 5; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///root/' }]); + const doc = tdm.setTextDocument( + 'file:///root/relative/main.ts', + 'typescript', + 'function hello() {\n return "world";\n}\n|' + ); + + const fakeHunk: RecentEdit = { + file: doc.uri, + startLine: 2, + endLine: 2, + diff: { + file: doc.uri, + pre: 1, + post: 3, + oldLen: 1, + newLen: 1, + before: [], + removed: [' return "world";'], + added: [' return "hello";'], + after: [], + } as DiffHunk, + timestamp: 1, + }; + mockRecentEditsProvider.getRecentEdits = () => [fakeHunk]; + + const snapshot = await createSnapshot(accessor, doc, '|'); + const text = querySnapshot(snapshot, 'RecentEdits.Chunk.Text') as string; + + assert.ok(text.includes('These are recently edited files. Do not suggest code that has been deleted.')); + assert.ok(text.includes('File: relative/main.ts')); + assert.ok(text.includes('@@ -2,1 +2,1 @@')); + assert.ok(text.includes('- return "world";')); + assert.ok(text.includes('+ return "hello";')); + assert.ok(text.includes('End of recent edits')); + }); + + test('renders recent edits correctly w/o deleted lines', async function () { + mockRecentEditsProvider.config.maxEdits = 5; + mockRecentEditsProvider.config.diffContextLines = 1; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = -1; + mockRecentEditsProvider.config.summarizationFormat = 'diff'; + mockRecentEditsProvider.config.maxFiles = 5; + mockRecentEditsProvider.config.removeDeletedLines = true; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///root/' }]); + const doc = tdm.setTextDocument( + 'file:///root/relative/main.ts', + 'typescript', + 'function hello() {\n return "world";\n}\n|' + ); + + const fakeHunk: RecentEdit = { + file: doc.uri, + startLine: 2, + endLine: 2, + diff: { + file: doc.uri, + pre: 1, + post: 3, + oldLen: 0, + newLen: 1, + before: [], + removed: [' return "world";'], + added: [' return "hello";'], + after: [], + } as DiffHunk, + timestamp: 1, + }; + mockRecentEditsProvider.getRecentEdits = () => [fakeHunk]; + + const snapshot = await createSnapshot(accessor, doc, '|'); + const text = querySnapshot(snapshot, 'RecentEdits.Chunk.Text') as string; + + assert.strictEqual( + text, + `These are recently edited files. Do not suggest code that has been deleted. +File: relative/main.ts +--- a/file:///root/relative/main.ts ++++ b/file:///root/relative/main.ts +@@ -2,1 +2,1 @@ ++ return "hello"; +End of recent edits\n`.replace(/\n {12}/g, '\n') + ); + }); + + test('limits the total number of open files from which to source edits', async function () { + mockRecentEditsProvider.config.maxEdits = 5; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = -1; + mockRecentEditsProvider.config.summarizationFormat = 'diff'; + mockRecentEditsProvider.config.maxFiles = 2; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///root/' }]); + + const fileUris = ['file:///root/file-1', 'file:///root/file-2', 'file:///root/file-3']; + for (const uri of fileUris) { + tdm.setTextDocument(uri, 'typescript', 'dummy\n|'); + } + const doc = tdm.setTextDocument('file:///root/relative/main.ts', 'typescript', 'dummy\n|'); + + const fakeHunks: RecentEdit[] = fileUris.map((uri, idx) => ({ + file: uri, + startLine: 1, + endLine: 1, + diff: { + file: uri, + pre: 0, + post: 1, + oldLen: 0, + newLen: 1, + before: [], + removed: [], + added: [`edit-${idx + 1}`], + after: [], + } as DiffHunk, + timestamp: idx + 1, + })); + mockRecentEditsProvider.getRecentEdits = () => fakeHunks; + + const snapshot = await createSnapshot(accessor, doc, '|'); + const text = querySnapshot(snapshot, 'RecentEdits.Chunk.Text') as string; + + assert.strictEqual( + text, + `These are recently edited files. Do not suggest code that has been deleted. +File: file-2 +--- a/file:///root/file-2 ++++ b/file:///root/file-2 +@@ -1,0 +1,1 @@ ++edit-2 +File: file-3 +--- a/file:///root/file-3 ++++ b/file:///root/file-3 +@@ -1,0 +1,1 @@ ++edit-3 +End of recent edits\n`.replace(/\n {12}/g, '\n') + ); + }); + + test('ignores edits over the max line limit', async function () { + mockRecentEditsProvider.config.diffContextLines = 1; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = -1; + mockRecentEditsProvider.config.summarizationFormat = 'diff'; + mockRecentEditsProvider.config.maxFiles = 10; + mockRecentEditsProvider.config.removeDeletedLines = true; + mockRecentEditsProvider.config.maxLinesPerEdit = 1; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///root/' }]); + + const fileUris = ['file:///root/file-1', 'file:///root/file-2', 'file:///root/file-3']; + for (const uri of fileUris) { + tdm.setTextDocument(uri, 'typescript', 'dummy\n|'); + } + const doc = tdm.setTextDocument('file:///root/relative/main.ts', 'typescript', 'dummy\n|'); + + const fakeHunks: RecentEdit[] = fileUris.map((uri, idx) => ({ + file: uri, + startLine: 1, + endLine: 1, + diff: { + file: uri, + pre: 0, + post: 1, + oldLen: 0, + newLen: 1, + before: [], + removed: [], + added: [`edit-${idx + 1}`], + after: [], + } as DiffHunk, + timestamp: idx + 1, + })); + + fakeHunks[0].diff.added.push('a second edit that breaks the 1 line limit'); + mockRecentEditsProvider.getRecentEdits = () => fakeHunks; + + const snapshot = await createSnapshot(accessor, doc, '|'); + const text = querySnapshot(snapshot, 'RecentEdits.Chunk.Text') as string; + + assert.strictEqual( + text, + `These are recently edited files. Do not suggest code that has been deleted. +File: file-2 +--- a/file:///root/file-2 ++++ b/file:///root/file-2 +@@ -1,0 +1,1 @@ ++edit-2 +File: file-3 +--- a/file:///root/file-3 ++++ b/file:///root/file-3 +@@ -1,0 +1,1 @@ ++edit-3 +End of recent edits\n`.replace(/\n {12}/g, '\n') + ); + }); + + test('returns none if too close to the cursor', async function () { + mockRecentEditsProvider.config.maxEdits = 5; + mockRecentEditsProvider.config.diffContextLines = 1; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = -1; + mockRecentEditsProvider.config.summarizationFormat = 'diff'; + mockRecentEditsProvider.config.maxFiles = 5; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = 3; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///root/' }]); + const doc = tdm.setTextDocument( + 'file:///root/relative/main.ts', + 'typescript', + 'function hello() {\n return "world";\n}\n|' + ); + + const fakeHunk: RecentEdit = { + file: doc.uri, + startLine: 2, + endLine: 2, + diff: { + file: doc.uri, + pre: 1, + post: 3, + oldLen: 1, + newLen: 1, + before: [], + removed: [' return "world";'], + added: [' return "hello";'], + after: [], + } as DiffHunk, + timestamp: 1, + }; + mockRecentEditsProvider.getRecentEdits = () => [fakeHunk]; + + const snapshot = await createSnapshot(accessor, doc, '|'); + assert.throws(() => querySnapshot(snapshot, 'RecentEdits')); + }); + + test('editIsTooCloseToCursor function returns true when edit directly intersects', function () { + const edit: RecentEdit = { + startLine: 2, + endLine: 2, + } as RecentEdit; + let filterByCursorLine = true; + let cursorLine = 1; + let activeDocDistanceLimitFromCursor = 1; + const editTooClose = editIsTooCloseToCursor( + edit, + filterByCursorLine, + cursorLine, + activeDocDistanceLimitFromCursor + ); + assert.strictEqual(editTooClose, true); + + cursorLine = 3; + activeDocDistanceLimitFromCursor = 4; + const editTooClose2 = editIsTooCloseToCursor( + edit, + filterByCursorLine, + cursorLine, + activeDocDistanceLimitFromCursor + ); + assert.strictEqual(editTooClose2, true); + + filterByCursorLine = false; + assert.strictEqual( + editIsTooCloseToCursor(edit, filterByCursorLine, cursorLine, activeDocDistanceLimitFromCursor), + false + ); + }); + + test('edits from content excluded documents are not included', async function () { + mockRecentEditsProvider.config.maxEdits = 5; + mockRecentEditsProvider.config.diffContextLines = 1; + mockRecentEditsProvider.config.activeDocDistanceLimitFromCursor = -1; + mockRecentEditsProvider.config.summarizationFormat = 'diff'; + mockRecentEditsProvider.config.maxFiles = 5; + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: 'file:///root/' }]); + + const doc = tdm.setTextDocument( + 'file:///root/relative/main.ts', + 'typescript', + 'function hello() {\n return "world";\n}\n|' + ); + const excludedDoc = tdm.setTextDocument( + 'file:///root/relative/excluded.ts', + 'typescript', + 'function excluded() {\n return "excluded";\n}\n|' + ); + + ignoreService.setBlockListUris([excludedDoc.uri]); + + const fakeEdits: RecentEdit[] = [ + { + file: doc.uri, + startLine: 2, + endLine: 2, + diff: { + file: doc.uri, + pre: 1, + post: 3, + oldLen: 1, + newLen: 1, + before: [], + removed: [' return "world";'], + added: [' return "hello";'], + after: [], + } as DiffHunk, + timestamp: 1, + }, + { + file: excludedDoc.uri, + startLine: 2, + endLine: 2, + diff: { + file: excludedDoc.uri, + pre: 1, + post: 3, + oldLen: 1, + newLen: 1, + before: [], + removed: [' return "world";'], + added: [' return "hello";'], + after: [], + } as DiffHunk, + timestamp: 1, + }, + ]; + mockRecentEditsProvider.getRecentEdits = () => fakeEdits; + + const snapshot = await createSnapshot(accessor, doc, '|'); + const text = querySnapshot(snapshot, 'RecentEdits.Chunk.Text') as string; + + assert.ok(text.includes('These are recently edited files. Do not suggest code that has been deleted.')); + assert.ok(text.includes('File: relative/main.ts')); + assert.ok(!text.includes('File: relative/excluded.ts')); + }); + + async function createSnapshot(accessor: ServicesAccessor, doc: CompletionRequestDocument, marker: string) { + const position = doc.positionAt(doc.getText().indexOf(marker)); + const tdms = accessor.get(ICompletionsTextDocumentManagerService); + const recentEditsProvider = accessor.get(ICompletionsRecentEditsProviderService); + const virtualPrompt = new VirtualPrompt(<RecentEdits tdms={tdms} recentEditsProvider={recentEditsProvider} />); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, doc, position)); + return virtualPrompt.snapshot().snapshot!; + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/similarFiles.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/similarFiles.test.tsx new file mode 100644 index 0000000..b19655b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/similarFiles.test.tsx @@ -0,0 +1,133 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { PromptSnapshotNode } from '../../../../../prompt/src/components/components'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { initializeTokenizers } from '../../../../../prompt/src/tokenization'; +import { CompletionRequestDocument } from '../../../prompt/completionsPromptFactory/componentsCompletionsPromptFactory'; +import { SimilarFiles } from '../../../prompt/components/similarFiles'; +import { CodeSnippetWithId, TraitWithId } from '../../../prompt/contextProviders/contextItemSchemas'; +import { NeighborSource } from '../../../prompt/similarFiles/neighborFiles'; +import { createCompletionRequestData } from '../../../test/completionsPrompt'; +import { createLibTestingContext } from '../../../test/context'; +import { querySnapshot } from '../../../test/snapshot'; +import { createTextDocument, TestTextDocumentManager } from '../../../test/textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; + +suite('Similar Files', function () { + let accessor: ServicesAccessor; + + setup(async function () { + accessor = createLibTestingContext().createTestingAccessor(); + NeighborSource.reset(); + await initializeTokenizers; + }); + + test('Empty render without similar file', async function () { + const doc = document('untitled:', 'typescript', 'const a = 23;'); + + const snapshot = await createSnapshot(accessor, doc, []); + + const snapshotNode = querySnapshot(snapshot, 'SimilarFiles') as PromptSnapshotNode[]; + assert.deepStrictEqual(snapshotNode, []); + }); + + test('Renders single similar file', async function () { + const doc = document('file:///foo.ts', 'typescript', '//sum\nconst result = |'); + const similarFile = document( + 'file:///calculator.ts', + 'typescript', + 'export function sum(a: number, b: number) { return a + b; }' + ); + + const snapshot = await createSnapshot(accessor, doc, [similarFile]); + + assert.deepStrictEqual( + querySnapshot(snapshot, 'SimilarFiles.f[0].SimilarFile.Chunk[0].Text'), + 'Compare this snippet from calculator.ts:' + ); + assert.deepStrictEqual( + querySnapshot(snapshot, 'SimilarFiles.f[0].SimilarFile.Chunk[1].Text'), + 'export function sum(a: number, b: number) { return a + b; }' + ); + }); + + test('Renders multiple similar files', async function () { + const doc = document('file:///foo.ts', 'typescript', '//sum and multiply\nconst result = |'); + const similar1 = document( + 'file:///sum.ts', + 'typescript', + 'export function sum(a: number, b: number) { return a + b; }' + ); + const similar2 = document( + 'file:///multiply.ts', + 'typescript', + 'export function multiply(a: number, b: number) { return a * b; }' + ); + + const snapshot = await createSnapshot(accessor, doc, [similar1, similar2]); + + const similarFileNodes = querySnapshot(snapshot, 'SimilarFiles') as PromptSnapshotNode[]; + assert.deepStrictEqual(similarFileNodes.length, 2); + assert.deepStrictEqual( + querySnapshot(snapshot, 'SimilarFiles.f[0].SimilarFile.Chunk[0].Text'), + 'Compare this snippet from sum.ts:' + ); + assert.deepStrictEqual( + querySnapshot(snapshot, 'SimilarFiles.f[0].SimilarFile.Chunk[1].Text'), + 'export function sum(a: number, b: number) { return a + b; }' + ); + assert.deepStrictEqual( + querySnapshot(snapshot, 'SimilarFiles.f[1].SimilarFile.Chunk[0].Text'), + 'Compare this snippet from multiply.ts:' + ); + assert.deepStrictEqual( + querySnapshot(snapshot, 'SimilarFiles.f[1].SimilarFile.Chunk[1].Text'), + 'export function multiply(a: number, b: number) { return a * b; }' + ); + }); + + test('Similar files can be turned off', async function () { + const doc = document('file:///foo.ts', 'typescript', '//sum\nconst result = |'); + const similarFile = document( + 'file:///calculator.ts', + 'typescript', + 'export function sum(a: number, b: number) { return a + b; }' + ); + + const snapshot = await createSnapshot(accessor, doc, [similarFile], undefined, undefined, true); + + const similarFiles = querySnapshot(snapshot, 'SimilarFiles') as PromptSnapshotNode[]; + assert.deepStrictEqual(similarFiles, []); + }); + + async function createSnapshot( + accessor: ServicesAccessor, + doc: CompletionRequestDocument, + neighbors: CompletionRequestDocument[], + codeSnippets?: CodeSnippetWithId[], + traits?: TraitWithId[], + turnOffSimilarFiles?: boolean, + ) { + const instantiationService = accessor.get(IInstantiationService); + const tdms = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + neighbors.forEach(n => tdms.setTextDocument(n.uri, n.detectedLanguageId, n.getText())); + const position = doc.positionAt(doc.getText().indexOf('|')); + + const virtualPrompt = new VirtualPrompt(<SimilarFiles tdms={tdms} instantiationService={instantiationService} />); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, doc, position, codeSnippets, traits, turnOffSimilarFiles)); + return virtualPrompt.snapshot().snapshot!; + } + + function document(uri: string, languageId: string, text: string) { + return createTextDocument(uri, languageId, 0, dedent`${text}`); + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/splitContextPromptRenderer.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/splitContextPromptRenderer.test.tsx new file mode 100644 index 0000000..37f153d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/splitContextPromptRenderer.test.tsx @@ -0,0 +1,1012 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import { CancellationTokenSource, Position } from 'vscode-languageserver-protocol'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { Chunk, PromptElementProps, PromptSnapshotNode, Text } from '../../../../../prompt/src/components/components'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { TokenizerName } from '../../../../../prompt/src/tokenization'; +import { AdditionalCompletionsContext, CompletionsContext } from '../../../prompt/components/completionsContext'; +import { CompletionsPromptRenderOptions } from '../../../prompt/components/completionsPromptRenderer'; +import { BeforeCursor, CurrentFile } from '../../../prompt/components/currentFile'; +import { SplitContextPromptRenderer } from '../../../prompt/components/splitContextPromptRenderer'; +import { CodeSnippetWithId, TraitWithId } from '../../../prompt/contextProviders/contextItemSchemas'; +import { createCompletionRequestData } from '../../../test/completionsPrompt'; +import { createLibTestingContext } from '../../../test/context'; +import { createTextDocument } from '../../../test/textDocument'; + +const MyNestedComponent = () => { + return ( + <> + <Text weight={0.5}>This goes first</Text> + <Text weight={0.6}>This goes last</Text> + </> + ); +}; + +const AnotherComponent = (props: PromptElementProps & { number: number }) => { + return <Text>This is a number {props.number ?? 0}</Text>; +}; + +const renderingOptions: CompletionsPromptRenderOptions = { + promptTokenLimit: 70, + suffixPercent: 20, + delimiter: '\n', + tokenizer: TokenizerName.o200k, + languageId: 'typescript', +}; + +const fullExpectedPrefixWithoutContext = 'const a = 1;\nfunction f'; +const fullExpectedContext = [ + 'This is a number 1\nThis goes first\nThis goes last\nThis is a number 2\nRaw text\nAnother raw text', +]; +const fullExpectedSuffix = 'const b = 2;\nconst c = 3;'; + +for (const lineEnding of ['\n', '\r\n']) { + const fileUri = 'file:///path/basename.ts'; + const source = `const a = 1;${lineEnding}function f|${lineEnding}const b = 2;${lineEnding}const c = 3;`; + const textDocument = createTextDocument(fileUri, 'typescript', 0, source); + const position: Position = textDocument.positionAt(textDocument.getText().indexOf('|')); + + suite(`Split context prompt renderer (line ending: ${JSON.stringify(lineEnding)})`, function () { + let accessor: ServicesAccessor; + let renderer: SplitContextPromptRenderer; + let snapshot: PromptSnapshotNode | undefined; + + setup(async function () { + accessor = createLibTestingContext().createTestingAccessor(); + renderer = new SplitContextPromptRenderer(); + const vPrompt = new VirtualPrompt( + ( + <> + <CompletionsContext> + <AnotherComponent number={1} /> + <MyNestedComponent /> + {/* This is intentionally placed here so that it's far from the other AnotherComponent*/} + <AnotherComponent number={2} /> + <> + {/* This is intentionally in a fragment to check that it's skipped */} + <Text>Raw text</Text> + </> + <> + <Text>Another raw text</Text> + </> + </CompletionsContext> + <CurrentFile /> + </> + ) + ); + const pipe = vPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + ({ snapshot } = vPrompt.snapshot()); + }); + + test('renders prefix, context and suffix based on completions doc position', function () { + const prompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, fullExpectedPrefixWithoutContext); + assert.deepStrictEqual(prompt.prefixTokens, 37); + assert.deepStrictEqual(prompt.suffix, fullExpectedSuffix); + assert.deepStrictEqual(prompt.suffixTokens, 12); + assert.deepStrictEqual(prompt.context, fullExpectedContext); + }); + + test('single context without comments', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.context, ['This is context']); + assert.deepStrictEqual(rendered.prefix, ''); + }); + + test('multiple context without comments', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.context, ['This is context\nThis is more context']); + assert.deepStrictEqual(rendered.prefix, ''); + }); + + test('multiple context blocks without comments', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <CompletionsContext> + <Text>This is other context</Text> + <Text>This is extra context</Text> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.context, [ + 'This is context\nThis is more context', + 'This is other context\nThis is extra context', + ]); + assert.deepStrictEqual(rendered.prefix, ''); + }); + + test('multiple types of context blocks without comments', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <AdditionalCompletionsContext> + <Text>This is other context</Text> + <Text>This is extra context</Text> + </AdditionalCompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.context, [ + 'This is context\nThis is more context', + 'This is other context\nThis is extra context', + ]); + assert.deepStrictEqual(rendered.prefix, ''); + }); + + test('multiple types of context blocks that are not adjacent', function () { + const prompt = ( + <> + <CompletionsContext> + <Text>This is context</Text> + <Text>This is more context</Text> + </CompletionsContext> + <CurrentFile /> + <AdditionalCompletionsContext> + <Text>This is other context</Text> + <Text>This is extra context</Text> + </AdditionalCompletionsContext> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const rendered = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.context, [ + 'This is context\nThis is more context', + 'This is other context\nThis is extra context', + ]); + assert.deepStrictEqual(rendered.prefix, ''); + }); + + test('uses configured tokenizer', function () { + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + tokenizer: TokenizerName.cl100k, + }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefixTokens, 37); + assert.deepStrictEqual(prompt.suffixTokens, 12); + }); + + test('computes metadata with stable updateDataTimeMs tolerance', function () { + const prompt1 = renderer.render(snapshot!, renderingOptions); + const prompt2 = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(prompt1.status, 'ok'); + assert.deepStrictEqual(prompt2.status, 'ok'); + + const metadata1 = prompt1.metadata; + const metadata2 = prompt2.metadata; + + assert.deepStrictEqual(metadata1.renderId, 0); + assert.deepStrictEqual(metadata2.renderId, 1); + assert.ok(metadata1.renderTimeMs > 0); + assert.ok(metadata1.elisionTimeMs > 0); + const expectedComponents = [ + { + componentPath: '$.f[1].CurrentFile', + }, + { + componentPath: '$.f[0].CompletionsContext[0].AnotherComponent[0].Text[0]', + expectedTokens: 7, + actualTokens: 7, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[1].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[2].AnotherComponent[0].Text[0]', + expectedTokens: 7, + actualTokens: 7, + }, + { + componentPath: '$.f[0].CompletionsContext[3].f[0].Text[0]', + expectedTokens: 3, + actualTokens: 3, + }, + { + componentPath: '$.f[0].CompletionsContext[4].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 12, + actualTokens: 12, + }, + ]; + + expectedComponents.forEach(expected => { + const actual = metadata1.componentStatistics.find(s => s.componentPath === expected.componentPath); + assert.ok(actual, `Component ${expected.componentPath} not found`); + assert.strictEqual(actual.expectedTokens, expected.expectedTokens); + assert.strictEqual(actual.actualTokens, expected.actualTokens); + // Instead of a fixed number, just ensure updateDataTimeMs is a non-negative number. + if (actual.updateDataTimeMs) { + assert.ok( + typeof actual.updateDataTimeMs === 'number' && actual.updateDataTimeMs >= 0, + `Expected updateDataTimeMs for ${expected.componentPath} to be a non-negative number` + ); + } + }); + }); + + test('computes usage statistics ignoring updateDataTimeMs field', function () { + const rendered = renderer.render(snapshot!, renderingOptions); + assert.deepStrictEqual(rendered.status, 'ok'); + const metadata = rendered.metadata; + // Make updateDataTimeMs a constant value to ensure it doesn't affect the test. + const actualStatsFiltered = metadata.componentStatistics.map(stats => { + if (stats.updateDataTimeMs) { + stats.updateDataTimeMs = 42; + } + return stats; + }); + const expectedStatsFiltered = [ + { + componentPath: '$.f[1].CurrentFile', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[0].AnotherComponent[0].Text[0]', + expectedTokens: 7, + actualTokens: 7, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[1].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[0].CompletionsContext[2].AnotherComponent[0].Text[0]', + expectedTokens: 7, + actualTokens: 7, + }, + { + componentPath: '$.f[0].CompletionsContext[3].f[0].Text[0]', + expectedTokens: 3, + actualTokens: 3, + }, + { + componentPath: '$.f[0].CompletionsContext[4].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 12, + actualTokens: 12, + }, + ]; + assert.deepStrictEqual(actualStatsFiltered, expectedStatsFiltered); + }); + + test('propagates source via statistics', function () { + const trait: TraitWithId = { + name: 'trait', + value: 'value', + id: 'traitid', + type: 'Trait', + }; + const codeSnippet: CodeSnippetWithId = { + uri: 'file://foo.ts', + value: 'value', + id: 'traitid', + type: 'CodeSnippet', + }; + const prompt = ( + <> + <CompletionsContext> + <Text source={trait}>This is a trait</Text> + <Chunk source={codeSnippet}> + <Text>This is a code snippet</Text> + </Chunk> + </CompletionsContext> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, renderingOptions); + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.ok(renderedPrompt.metadata.componentStatistics.find(s => s.source === trait)); + assert.ok(renderedPrompt.metadata.componentStatistics.find(s => s.source === codeSnippet)); + }); + + test('elides prefix', function () { + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 5, + suffixPercent: 0, + }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, 'function f'); + assert.deepStrictEqual(prompt.suffix, ''); + }); + + test('elides context', function () { + const prompt = renderer.render( + snapshot!, + { + ...renderingOptions, + promptTokenLimit: 30, + suffixPercent: 0, + }, + undefined + ); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.prefix, fullExpectedPrefixWithoutContext); + assert.deepStrictEqual(prompt.context, [ + 'This is a number 1\nThis is a number 2\nRaw text\nAnother raw text', + ]); + assert.deepStrictEqual(prompt.suffix, ''); + }); + + test('elides suffix (from the end!)', function () { + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 30, + suffixPercent: 10, + }); + + assert.deepStrictEqual(prompt.status, 'ok'); + assert.deepStrictEqual(prompt.suffix, 'const b ='); + }); + + test('elides context and suffix partially', function () { + // Use tighter token limits to force partial elision on both sides. + const prompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 20, + suffixPercent: 10, + }); + // We don't have the exact expected strings, but we verify that both context and suffix + // have been elided compared to the full expectations. + assert.strictEqual(prompt.status, 'ok'); + // The elided prefix should be shorter than the full expected one. + assert.ok(prompt.context![0].length < fullExpectedContext[0].length, 'Expected context to be elided'); + // The elided suffix should also be shorter than the full expected suffix, if any elision took place. + if (fullExpectedSuffix.length > 0) { + assert.ok(prompt.suffix.length < fullExpectedSuffix.length, 'Expected suffix to be elided'); + } + }); + + test('generates prompt metadata', function () { + const rendered = renderer.render(snapshot!, renderingOptions); + assert.deepStrictEqual(rendered.status, 'ok'); + const metadata = rendered.metadata; + assert.ok(metadata.renderId === 0); + assert.ok(metadata.elisionTimeMs > 0); + assert.ok(metadata.renderTimeMs > 0); + assert.ok(metadata.updateDataTimeMs > 0); + assert.deepStrictEqual(metadata.tokenizer, TokenizerName.o200k); + }); + + test('computes usage statistics after elision', function () { + const rendered = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 40, + suffixPercent: 10, + }); + assert.deepStrictEqual(rendered.status, 'ok'); + const metadata = rendered.metadata; + const actualStatsFiltered = metadata.componentStatistics.map(stats => { + if (stats.updateDataTimeMs) { + stats.updateDataTimeMs = 42; + } + return stats; + }); + assert.deepStrictEqual( + metadata.componentStatistics.reduce((acc, stats) => acc + (stats.actualTokens ?? 0), 0), + 33 + ); + assert.deepStrictEqual(actualStatsFiltered, [ + { + componentPath: '$.f[1].CurrentFile', + updateDataTimeMs: 42, + }, + { + componentPath: '$.f[0].CompletionsContext[0].AnotherComponent[0].Text[0]', + expectedTokens: 7, + actualTokens: 7, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 0, + }, + { + componentPath: '$.f[0].CompletionsContext[1].MyNestedComponent[0].f[1].Text[0]', + expectedTokens: 4, + actualTokens: 0, + }, + { + componentPath: '$.f[0].CompletionsContext[2].AnotherComponent[0].Text[0]', + expectedTokens: 7, + actualTokens: 7, + }, + { + componentPath: '$.f[0].CompletionsContext[3].f[0].Text[0]', + expectedTokens: 3, + actualTokens: 3, + }, + { + componentPath: '$.f[0].CompletionsContext[4].f[0].Text[0]', + expectedTokens: 4, + actualTokens: 4, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[0].BeforeCursor[0].Text[0]', + expectedTokens: 8, + actualTokens: 8, + }, + { + componentPath: '$.f[1].CurrentFile[0].f[1].AfterCursor[0].Text[0]', + expectedTokens: 12, + actualTokens: 4, + }, + ]); + }); + + function createStringWithNLines(n: number, baseText: string): string { + let result = ''; + for (let i = 1; i <= n; i++) { + result += `${baseText}${i}\n`; + } + return result; + } + + test('uses cached suffix if similar enough', async function () { + const firstSuffix = createStringWithNLines(15, 'a') + createStringWithNLines(10, 'b'); + const secondSuffix = createStringWithNLines(15, 'a') + createStringWithNLines(10, 'c'); + const renderOptionsWithSuffix: CompletionsPromptRenderOptions = { + ...renderingOptions, + promptTokenLimit: 205, + suffixPercent: 50, + }; + const textDocumentWithFirstSuffix = createTextDocument( + fileUri, + 'typescript', + 0, + 'function f|\n' + firstSuffix + ); + const position = textDocumentWithFirstSuffix.positionAt(textDocumentWithFirstSuffix.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithFirstSuffix, position)); + // Snapshot caches the suffix + virtualPrompt.snapshot(); + + // The position is the same, since the start of the document doesn't change + const textDocumentWithSecondSuffix = createTextDocument( + fileUri, + 'typescript', + 1, + 'function f|\n' + secondSuffix + ); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithSecondSuffix, position)); + const { snapshot: snapshotWithDefaultThreshold } = virtualPrompt.snapshot(); + + // the first suffix is used, since they are similar enough + const renderedWithDefaultThreshold = renderer.render( + snapshotWithDefaultThreshold!, + renderOptionsWithSuffix + ); + assert.deepStrictEqual(renderedWithDefaultThreshold.status, 'ok'); + assert.deepStrictEqual(renderedWithDefaultThreshold.suffix, firstSuffix); + + await pipe.pump( + createCompletionRequestData( + accessor, + textDocumentWithSecondSuffix, + position, + undefined, + undefined, + undefined, + 3 + ) + ); + const { snapshot: snapshotWithLowerThreshold } = virtualPrompt.snapshot(); + + // The second suffix is used, since the matching threshold is lower + const renderedWithLowerThreshold = renderer.render(snapshotWithLowerThreshold!, renderOptionsWithSuffix); + assert.deepStrictEqual(renderedWithLowerThreshold.status, 'ok'); + assert.deepStrictEqual(renderedWithLowerThreshold.suffix, secondSuffix); + }); + + test('does not use cached suffix if not similar enough', async function () { + const firstSuffix = createStringWithNLines(15, 'a') + createStringWithNLines(10, 'b'); + const secondSuffix = createStringWithNLines(3, 'a') + createStringWithNLines(22, 'c'); + const renderOptionsWithSuffix: CompletionsPromptRenderOptions = { + ...renderingOptions, + promptTokenLimit: 205, + suffixPercent: 50, + }; + const textDocumentWithFirstSuffix = createTextDocument( + fileUri, + 'typescript', + 0, + 'function f|\n' + firstSuffix + ); + const position = textDocumentWithFirstSuffix.positionAt(textDocumentWithFirstSuffix.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithFirstSuffix, position)); + // Snapshot caches the suffix + virtualPrompt.snapshot(); + + // The position is the same, since the start of the document doesn't change + const textDocumentWithSecondSuffix = createTextDocument( + fileUri, + 'typescript', + 1, + 'function f|\n' + secondSuffix + ); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithSecondSuffix, position)); + const { snapshot } = virtualPrompt.snapshot(); + + // the second suffix is used, since they are not similar enough + const rendered = renderer.render(snapshot!, renderOptionsWithSuffix); + assert.deepStrictEqual(rendered.status, 'ok'); + assert.deepStrictEqual(rendered.suffix, secondSuffix); + }); + + test('suffix can be empty', async function () { + const textDocumentWithoutSuffix = createTextDocument(fileUri, 'typescript', 0, 'function f|'); + const position = textDocumentWithoutSuffix.positionAt(textDocumentWithoutSuffix.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, textDocumentWithoutSuffix, position)); + const { snapshot } = virtualPrompt.snapshot(); + const promptWithoutSuffix = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(promptWithoutSuffix.status, 'ok'); + assert.deepStrictEqual(promptWithoutSuffix.suffix, ''); + assert.deepStrictEqual(promptWithoutSuffix.prefix, 'function f'); + }); + + test('prefix can be empty', async function () { + const emptyTextDocument = createTextDocument(fileUri, 'typescript', 0, '|\nconst b = 2;'); + const position = emptyTextDocument.positionAt(emptyTextDocument.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, emptyTextDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const emptyPrompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(emptyPrompt.status, 'ok'); + assert.deepStrictEqual(emptyPrompt.prefix, ''); + assert.deepStrictEqual(emptyPrompt.suffix, 'const b = 2;'); + }); + + test('prefix and suffix can be empty', async function () { + const emptyTextDocument = createTextDocument(fileUri, 'typescript', 0, ''); + const position = emptyTextDocument.positionAt(0); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + await pipe.pump(createCompletionRequestData(accessor, emptyTextDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const emptyPrompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(emptyPrompt.status, 'ok'); + assert.deepStrictEqual(emptyPrompt.prefix, ''); + assert.deepStrictEqual(emptyPrompt.suffix, ''); + }); + + test('cancels rendering when token has been cancelled', function () { + const cts = new CancellationTokenSource(); + + cts.cancel(); + const prompt = renderer.render(snapshot!, renderingOptions, cts.token); + + assert.deepStrictEqual(prompt.status, 'cancelled'); + }); + + test('throws error when tree does not contain completions document component', function () { + const promptCompletionsDocument = ( + <> + <Text>Whatever</Text> + </> + ); + + const virtualPrompt = new VirtualPrompt(promptCompletionsDocument); + const { snapshot } = virtualPrompt.snapshot(); + const prompt = renderer.render(snapshot!, renderingOptions); + + assert.strictEqual(prompt.status, 'error'); + assert.strictEqual(prompt.error.message, `Node of type ${BeforeCursor.name} not found`); + }); + + test('renders empty prefix and suffix if no data is sent', function () { + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const emptyPrompt = renderer.render(snapshot!, renderingOptions); + + assert.deepStrictEqual(emptyPrompt.status, 'ok'); + assert.deepStrictEqual(emptyPrompt.prefix, ''); + assert.deepStrictEqual(emptyPrompt.suffix, ''); + }); + + test('does not re-render if no data matching the expected structure is sent', async function () { + const textDocument = createTextDocument( + fileUri, + 'typescript', + 0, + `import * from './foo.ts'\n|\nfunction f` + ); + const position = textDocument.positionAt(textDocument.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + + // First render + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const renderedPrompt = renderer.render(snapshot!, renderingOptions); + + // Second render + const { snapshot: snapshotTwo } = virtualPrompt.snapshot(); + const renderedPromptTwo = renderer.render(snapshotTwo!, renderingOptions); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPromptTwo.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, `import * from './foo.ts'\n`); + assert.deepStrictEqual(renderedPrompt.prefix, renderedPromptTwo.prefix); + assert.deepStrictEqual(renderedPrompt.suffix, 'function f'); + assert.deepStrictEqual(renderedPrompt.suffix, renderedPromptTwo.suffix); + assert.deepStrictEqual(renderedPrompt.prefixTokens, renderedPromptTwo.prefixTokens); + assert.deepStrictEqual(renderedPrompt.suffixTokens, renderedPromptTwo.suffixTokens); + }); + + test('re-renders if new data matching the expected structure is sent', async function () { + const textDocument = createTextDocument( + fileUri, + 'typescript', + 0, + `import * from './foo.ts'\n|\nfunction f` + ); + const position = textDocument.positionAt(textDocument.getText().indexOf('|')); + const prompt = ( + <> + <CurrentFile /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const pipe = virtualPrompt.createPipe(); + + // First render + await pipe.pump(createCompletionRequestData(accessor, textDocument, position)); + const { snapshot } = virtualPrompt.snapshot(); + const renderedPrompt = renderer.render(snapshot!, renderingOptions); + + // Second render + const updatedTextDocument = createTextDocument( + fileUri, + 'typescript', + 1, // Notice version change + `import * from './bar.ts'\n|\nfunction g` + ); + const updatedPosition = updatedTextDocument.positionAt(updatedTextDocument.getText().indexOf('|')); + + await pipe.pump(createCompletionRequestData(accessor, updatedTextDocument, updatedPosition)); + const { snapshot: snapshotTwo } = virtualPrompt.snapshot(); + const renderedPromptTwo = renderer.render(snapshotTwo!, renderingOptions); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPromptTwo.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.prefix, `import * from './foo.ts'\n`); + assert.deepStrictEqual(renderedPromptTwo.prefix, `import * from './bar.ts'\n`); + assert.deepStrictEqual(renderedPrompt.suffix, 'function f'); + assert.deepStrictEqual(renderedPromptTwo.suffix, 'function g'); + }); + + test('Elides Chunk completely', function () { + const prompt = ( + <> + <CompletionsContext> + <Chunk weight={0.5}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text>Outside Text</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 10, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.context, ['Outside Text']); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('Elides Chunk completely while respecting lower weights', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.5}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 16, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.context, ['Outside Text 1\nOutside Text 2']); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('Elides Chunk completely in case of exceeding the limit even with higher weight', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.5}>Outside Text 1</Text> + <Chunk weight={0.7}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.8}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 14, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.context, ['Outside Text 1\nOutside Text 2']); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('Prefers higher weighted Chunk over lower weighted separate components', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.8}> + <Text>Chunk Text 1</Text> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 14, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.context, ['Chunk Text 1\nChunk Text 2']); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('If a nested chunk is elided first, the outer chunks is kept', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.5}> + <Text>Chunk Text 1</Text> + <Chunk weight={0.5}> + <Text>Nested Chunk Text 1</Text> + <Text>Nested Chunk Text 2</Text> + </Chunk> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 30, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.context, [ + 'Outside Text 1\nChunk Text 1\nChunk Text 2\nOutside Text 2', + ]); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + + test('If the outer chunk is elided first, the inner chunk is also elided', function () { + const prompt = ( + <> + <CompletionsContext> + <Text weight={0.7}>Outside Text 1</Text> + <Chunk weight={0.5}> + <Text weight={0.5}>Chunk Text 1</Text> + <Chunk> + <Text>Nested Chunk Text 1</Text> + <Text>Nested Chunk Text 2</Text> + </Chunk> + <Text>Chunk Text 2</Text> + </Chunk> + <Text weight={0.7}>Outside Text 2</Text> + </CompletionsContext> + <CurrentFile weight={0.9} /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + + const renderedPrompt = renderer.render(snapshot!, { + ...renderingOptions, + promptTokenLimit: 30, + suffixPercent: 0, + }); + + assert.deepStrictEqual(renderedPrompt.status, 'ok'); + assert.deepStrictEqual(renderedPrompt.context, ['Outside Text 1\nOutside Text 2']); + assert.deepStrictEqual(renderedPrompt.suffix, ''); + }); + }); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/test/traits.test.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/test/traits.test.tsx new file mode 100644 index 0000000..2b1ef06 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/test/traits.test.tsx @@ -0,0 +1,125 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../../prompt/jsx-runtime/ */ + +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { PromptSnapshotNode } from '../../../../../prompt/src/components/components'; +import { VirtualPrompt } from '../../../../../prompt/src/components/virtualPrompt'; +import { extractNodesWitPath } from '../../../../../prompt/src/test/components/testHelpers'; +import { CompletionRequestData } from '../../../prompt/completionsPromptFactory/componentsCompletionsPromptFactory'; +import { Traits } from '../../../prompt/components/traits'; +import { TraitWithId } from '../../../prompt/contextProviders/contextItemSchemas'; +import { TelemetryWithExp } from '../../../telemetry'; +import { createLibTestingContext } from '../../../test/context'; +import { querySnapshot } from '../../../test/snapshot'; +import { createTextDocument } from '../../../test/textDocument'; + +suite('Traits component', function () { + let accessor: ServicesAccessor; + + const trait1: TraitWithId = { + name: 'foo', + value: 'bar', + importance: 10, + id: 'traitid1', + type: 'Trait', + }; + const trait2: TraitWithId = { + name: 'baz', + value: 'qux', + importance: 5, + id: 'traitid2', + type: 'Trait', + }; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + }); + + test('Renders nothing if there are no traits', async function () { + try { + await renderTrait(accessor); + } catch (e) { + assert.ok((e as Error).message.startsWith('No children found at path segment ')); + } + }); + + test('Renders nothing if the traits array is empty', async function () { + try { + await renderTrait(accessor, []); + } catch (e) { + assert.ok((e as Error).message.startsWith('No children found at path segment ')); + } + }); + + test('Renders a single trait', async function () { + const snapshot = await renderTrait(accessor, [trait1]); + const traits = querySnapshot(snapshot.snapshot!, 'Traits') as PromptSnapshotNode[]; + assert.deepStrictEqual(traits.length, 2); + assert.deepStrictEqual(traits[0].children?.[0].value, 'Consider this related information:\n'); + assert.deepStrictEqual(traits[1].props?.source, trait1); + assert.deepStrictEqual(traits[1].children?.[0].value, 'foo: bar'); + }); + + test('Renders multiple traits', async function () { + const snapshot = await renderTrait(accessor, [trait1, trait2]); + const result = querySnapshot(snapshot.snapshot!, 'Traits') as PromptSnapshotNode[]; + + // Assert that keys are in the path + assert.deepStrictEqual(extractNodesWitPath(snapshot.snapshot!), [ + '$[0].Traits', + '$[0].Traits[0].f', + '$[0].Traits[0].f[0].Text', + '$[0].Traits[0].f[0].Text[0]', + '$[0].Traits[0].f["traitid1"].Text', + '$[0].Traits[0].f["traitid1"].Text[0]', + '$[0].Traits[0].f["traitid2"].Text', + '$[0].Traits[0].f["traitid2"].Text[0]', + ]); + + assert.deepStrictEqual(result.length, 3); + const traits = querySnapshot(snapshot.snapshot!, 'Traits') as PromptSnapshotNode[]; + assert.deepStrictEqual(traits.length, 3); + assert.deepStrictEqual(traits[0].children?.[0].value, 'Consider this related information:\n'); + assert.deepStrictEqual(traits[1].props?.source, trait1); + assert.deepStrictEqual(traits[1].children?.[0].value, 'foo: bar'); + assert.deepStrictEqual(traits[2].props?.source, trait2); + assert.deepStrictEqual(traits[2].children?.[0].value, 'baz: qux'); + }); +}); + +async function renderTrait(accessor: ServicesAccessor, traits?: TraitWithId[]) { + const document = createTextDocument( + 'file:///foo.ts', + 'typescript', + 0, + dedent` + const a = 1; + function f| + const b = 2; + ` + ); + const position = document.positionAt(document.getText().indexOf('|')); + + const virtualPrompt = new VirtualPrompt(<Traits />); + const pipe = virtualPrompt.createPipe(); + + const completionRequestData: CompletionRequestData = { + document, + position, + telemetryData: TelemetryWithExp.createEmptyConfigForTesting(), + cancellationToken: new CancellationTokenSource().token, + maxPromptTokens: 1000, + data: undefined, + traits, + }; + + await pipe.pump(completionRequestData); + return virtualPrompt.snapshot(); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/traits.tsx b/completions-sample-code/vscode-node/lib/src/prompt/components/traits.tsx new file mode 100644 index 0000000..9d6d6cd --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/traits.tsx @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../../prompt/jsx-runtime/ */ + +import { ComponentContext, PromptElementProps, Text } from '../../../../prompt/src/components/components'; +import { normalizeLanguageId } from '../../../../prompt/src/prompt'; +import { + CompletionRequestData, + isCompletionRequestData, +} from '../completionsPromptFactory/componentsCompletionsPromptFactory'; +import { TraitWithId } from '../contextProviders/contextItemSchemas'; + +export const Traits = (_props: PromptElementProps, context: ComponentContext) => { + const [traits, setTraits] = context.useState<TraitWithId[]>(); + const [languageId, setLanguageId] = context.useState<string>(); + + context.useData(isCompletionRequestData, (data: CompletionRequestData) => { + if (data.traits !== traits) { + setTraits(data.traits); + } + + const normalizedLanguageId = normalizeLanguageId(data.document.detectedLanguageId); + if (normalizedLanguageId !== languageId) { + setLanguageId(normalizedLanguageId); + } + }); + + if (!traits || traits.length === 0 || !languageId) { + return; + } + + // TODO: use a `KeepTogether` elision that removes the header if no traits are present + return ( + <> + <Text>{'Consider this related information:\n'}</Text> + {...traits.map(trait => ( + <Text key={trait.id} source={trait}> + {`${trait.name}: ${trait.value}`} + </Text> + ))} + </> + ); +}; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/components/virtualComponent.ts b/completions-sample-code/vscode-node/lib/src/prompt/components/virtualComponent.ts new file mode 100644 index 0000000..ea8fcc6 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/components/virtualComponent.ts @@ -0,0 +1,251 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ComponentStatistics, PromptMetadata } from '../../../../prompt/src/components/components'; +import { getTokenizer, Tokenizer, TokenizerName } from '../../../../prompt/src/tokenization'; +import { LRUCacheMap } from '../../helpers/cache'; +import { setDefault } from '../../util/map'; +import { CompletionsPromptOptions } from '../completionsPromptFactory/completionsPromptFactory'; +import { CodeSnippetWithId, TraitWithId } from '../contextProviders/contextItemSchemas'; +import { + EMPTY_NODE, + IVirtualNode, + NodeId, + rectifyWeights, + render, + RenderedText, + RenderNode, + snapshot, +} from '../render/renderNode'; +import { getAvailableNodeId, NodeCostFunction } from '../render/utils'; + +/* How many lines of prefix/suffix should have cached token costs */ +const NUM_CACHED_LINE_COSTS = 20_000; + +export type RenderedComponent = { + root: RenderNode; + renderedNodes: Map<NodeId, RenderNode>; + text: string; + cost: number; + metadata: PromptMetadata; +}; + +export type ComponentSnapshot = { + root: RenderNode; + // A set of node IDs to exclude from rendering (e.g. because they are already included elsewhere in the prompt, or are invalid for this request). + mask?: NodeId[]; + // Nodes to be tracked for telemetry statistics + statistics?: Map<NodeId, ComponentStatistics>; +}; + +export type ValidatedContextItems = { + traits: TraitWithId[]; + codeSnippets: CodeSnippetWithId[]; +}; + +export interface VirtualPromptComponent { + name: string; + snapshot(options: CompletionsPromptOptions, context?: ValidatedContextItems): ComponentSnapshot; + estimatedCost?(options: CompletionsPromptOptions, context?: ValidatedContextItems): number; +} + +let renderId = 0; // Unique across all render calls, used for telemetry +const renderCache = new LRUCacheMap< + NodeId, + { budget: number; mask: Set<NodeId>; tokenizer: TokenizerName; render: RenderedText } +>(); +export function renderWithMetadata( + component: VirtualPromptComponent, + budget: number, + options: CompletionsPromptOptions, + context?: ValidatedContextItems +): RenderedComponent { + renderId++; + const tokenizerName = options.promptOpts?.tokenizer ?? TokenizerName.o200k; + const start = performance.now(); + const { root, mask, statistics } = component.snapshot(options, context); + const renderEnd = performance.now(); + + const maskSet = new Set(mask); + const cachedRender = renderCache?.get(root.id); + let renderedText: RenderedText; + if ( + cachedRender && + cachedRender.budget >= budget && + cachedRender.render.cost <= budget && + cachedRender.tokenizer === tokenizerName && + maskSet.size === cachedRender.mask.size && + [...maskSet].every(id => cachedRender.mask.has(id)) + ) { + // If we have a cached render, use it if we expect the same result + // (identical masks and tokenizer, cost within budget, and previous budget at least as large as current budget) + renderedText = cachedRender.render; + } else { + // Otherwise, render the node + const tokenizer = getTokenizer(tokenizerName); + const costFunction = (text: string) => tokenizer.tokenLength(text); + renderedText = render(root, { budget, mask, costFunction }); + renderCache.set(root.id, { + budget, + mask: maskSet, + tokenizer: tokenizerName, + render: renderedText, + }); + } + const { text, cost, renderedNodes } = renderedText; + const elisionEnd = performance.now(); + for (const [id, stat] of statistics?.entries() ?? []) { + // Note that we are currently only recording the cost of the node itself, not the costs of its children. + // This is enough for existing telemetry, since we put CodeSnippets and Traits in their own nodes. + stat.actualTokens = renderedNodes.get(id)?.cost ?? 0; + } + const metadata: PromptMetadata = { + renderId: renderId, + rendererName: 'renderNode', + tokenizer: tokenizerName, + elisionTimeMs: elisionEnd - renderEnd, + renderTimeMs: renderEnd - start, + updateDataTimeMs: 0, + componentStatistics: [{ componentPath: component.name, actualTokens: cost }], + }; + return { root, renderedNodes, text, cost, metadata }; +} + +function cachedLineCostFunction(tokenizer: Tokenizer, cache: Map<string, number>): NodeCostFunction { + return (node: IVirtualNode) => { + const key = node.text.join('') + '\n'; + // since actual token costs aren't known until we concatenate the lines, + // we slightly overestimate the cost to increase likelihood of respecting budget on first try + return setDefault(cache, key, () => tokenizer.tokenLength(key) + 1); + }; +} + +function getLinewiseNode(raw: string, costFunction: NodeCostFunction, reversed: boolean): RenderNode { + const lines = raw.split('\n'); + const children = lines.map(line => ({ id: getAvailableNodeId(), text: [line], children: [], canMerge: true })); + const seps = ['']; + if (children.length >= 1) { + seps.push(...Array<string>(children.length - 1).fill('\n'), ''); + } + const virtualNode = { id: getAvailableNodeId(), text: seps, children, canMerge: true }; + // Don't include elision marker in node cost, since there will be at most one such marker + const nodeCostFunction = (node: IVirtualNode) => (node.id === virtualNode.id ? 0 : costFunction(node)); + const root = snapshot(virtualNode, nodeCostFunction); + // Weight lines so that each line is has less value than the following one + // (Or more value, if reversed) + let valueTarget = reversed ? children.length : 1; + for (const child of root.children) { + child.weight = valueTarget * Math.max(1, child.cost); + valueTarget += reversed ? -1 : 1; + } + return root; +} + +export class BasicPrefixComponent implements VirtualPromptComponent { + readonly name = 'basicPrefix'; + private costCache = new LRUCacheMap<string, number>(NUM_CACHED_LINE_COSTS); + + snapshot(options: CompletionsPromptOptions): ComponentSnapshot { + const { completionState, promptOpts } = options; + const rawPrefix = completionState.textDocument.getText({ + start: { line: 0, character: 0 }, + end: completionState.position, + }); + const tokenizer = getTokenizer(promptOpts?.tokenizer); + const costFunction = cachedLineCostFunction(tokenizer, this.costCache); + const root = getLinewiseNode(rawPrefix, costFunction, false); + return { root }; + } +} + +export class TraitComponent implements VirtualPromptComponent { + readonly name = 'traitProvider'; + + snapshot(options: CompletionsPromptOptions, context?: ValidatedContextItems): ComponentSnapshot { + const { promptOpts } = options; + const tokenizer = getTokenizer(promptOpts?.tokenizer); + if (!context || context.traits.length === 0) { + return { root: EMPTY_NODE }; + } + const weights: Map<number, number> = new Map(); + let totalWeight = 0; + const children: RenderNode[] = []; + const statistics: Map<NodeId, ComponentStatistics> = new Map(); + for (const trait of context.traits) { + const id = getAvailableNodeId(); + const text = `${trait.name}: ${trait.value}`; + const child: RenderNode = { + id, + text: [text], + children: [], + cost: tokenizer.tokenLength(text), + weight: 0, + elisionMarker: '', + canMerge: true, + requireRenderedChild: true, + }; + children.push(child); + statistics.set(id, { + componentPath: trait.id, + source: trait, + expectedTokens: child.cost, + }); + weights.set(id, trait.importance ?? 0); + totalWeight += trait.importance ?? 0; + } + totalWeight = Math.max(totalWeight, 1); + const header = `Related context:\n`; + const text: string[] = [header, ...new Array<string>(children.length).fill('\n')]; + const root: RenderNode = { + id: getAvailableNodeId(), + text, + children, + cost: 0, + weight: 0, + elisionMarker: '', + canMerge: true, + requireRenderedChild: true, + }; + rectifyWeights(root, node => (weights.get(node.id) ?? 0) / totalWeight); + return { root, statistics }; + } +} + +export class ConcatenatedContextComponent implements VirtualPromptComponent { + constructor( + readonly name: string, + readonly components: VirtualPromptComponent[] + ) { } + + snapshot(options: CompletionsPromptOptions, context?: ValidatedContextItems): ComponentSnapshot { + const snapshots = this.components.map(component => component.snapshot(options, context)); + const children = snapshots.map(s => s.root).filter(n => n.id !== EMPTY_NODE.id); + if (children.length === 0) { + return { root: EMPTY_NODE }; + } + const text = ['', ...Array<string>(children.length - 1).fill('\n'), '']; + const root: RenderNode = { + id: getAvailableNodeId(), + text, + children, + cost: 0, + weight: 0, + elisionMarker: '', + canMerge: true, + requireRenderedChild: false, + }; + const mask: NodeId[] = []; + const statistics = new Map<NodeId, ComponentStatistics>(); + for (const s of snapshots) { + for (const [id, stat] of s.statistics?.entries() ?? []) { + statistics.set(id, stat); + } + if (s.mask) { + mask.push(...s.mask); + } + } + return { root, mask, statistics }; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistry.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistry.ts new file mode 100644 index 0000000..8380e2d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistry.ts @@ -0,0 +1,539 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken, CancellationTokenSource, DocumentSelector } from 'vscode-languageserver-protocol'; +import { ILanguageContextProviderService, ProviderTarget } from '../../../../../../platform/languageContextProvider/common/languageContextProviderService'; +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { isCancellationError } from '../../../../../../util/vs/base/common/errors'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + ContextItemUsageDetails, + ContextProvider, + DocumentContext, + ResolutionStatus, + ResolveRequest, + ResolveResult, + SupportedContextItem, + UsageStatus, +} from '../../../types/src'; +import { ConfigKey, getConfig } from '../config'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { LRUCacheMap } from '../helpers/cache'; +import { ICompletionsLogTargetService, logger } from '../logger'; +import { TelemetryWithExp } from '../telemetry'; +import { ICompletionsRuntimeModeService } from '../util/runtimeMode'; +import { isArrayOfT, resolveAll } from './asyncUtils'; +import { fillInCppVSCodeActiveExperiments } from './contextProviderRegistryCpp'; +import { fillInCSharpActiveExperiments } from './contextProviderRegistryCSharp'; +import { fillInMultiLanguageActiveExperiments } from './contextProviderRegistryMultiLanguage'; +import { fillInTsActiveExperiments } from './contextProviderRegistryTs'; +import { + addOrValidateContextItemsIDs, + filterSupportedContextItems, + SupportedContextItemWithId, +} from './contextProviders/contextItemSchemas'; +import { ICompletionsContextProviderService } from './contextProviderStatistics'; + +export interface ResolvedContextItem<T extends SupportedContextItemWithId = SupportedContextItemWithId> { + providerId: string; + matchScore: number; + resolution: ResolutionStatus; + resolutionTimeMs: number; + data: T[]; +} + +export interface ContextProviderTelemetry { + providerId: string; + matched: boolean; + resolution: ResolutionStatus; + resolutionTimeMs: number; + usage: UsageStatus; + usageDetails?: ContextItemUsageDetails[]; + numResolvedItems: number; + numUsedItems?: number; + numPartiallyUsedItems?: number; +} + +export const ICompletionsContextProviderRegistryService = createServiceIdentifier<ICompletionsContextProviderRegistryService>('ICompletionsContextProviderRegistryService'); +export interface ICompletionsContextProviderRegistryService { + readonly _serviceBrand: undefined; + + registerContextProvider<T extends SupportedContextItem>(provider: ContextProvider<T>): void; + unregisterContextProvider(providerId: string): void; + providers: ContextProvider<SupportedContextItem>[]; + resolveAllProviders( + completionId: string, + opportunityId: string, + documentContext: DocumentContext, + telemetryData: TelemetryWithExp, + completionToken?: CancellationToken, + data?: unknown + ): Promise<ResolvedContextItem[]>; +} + +export type ActiveExperiments = Map<string, string | number | boolean | string[]>; + +export const ICompletionsDefaultContextProviders = createServiceIdentifier<ICompletionsDefaultContextProviders>('ICompletionsDefaultContextProviders'); +export interface ICompletionsDefaultContextProviders { + readonly _serviceBrand: undefined; + getIds(): string[]; + add(id: string): void; +} + +export class DefaultContextProvidersContainer implements ICompletionsDefaultContextProviders { + declare _serviceBrand: undefined; + + private ids: string[] = []; + + add(id: string) { + this.ids.push(id); + } + + getIds(): string[] { + return this.ids; + } +} + +type ContextProviderMatchFunction = ( + instantiationService: IInstantiationService, + documentSelector: DocumentSelector, + documentContext: DocumentContext +) => Promise<number> | number; + + +export class CoreContextProviderRegistry implements ICompletionsContextProviderRegistryService { + declare _serviceBrand: undefined; + + constructor( + private match: ContextProviderMatchFunction, + @ILanguageContextProviderService private registryService: ILanguageContextProviderService, + @ICompletionsRuntimeModeService private runtimeMode: ICompletionsRuntimeModeService, + @IInstantiationService protected instantiationService: IInstantiationService, + @ICompletionsLogTargetService protected logTarget: ICompletionsLogTargetService, + @ICompletionsContextProviderService protected contextProviderStatistics: ICompletionsContextProviderService, + ) { } + + registerContextProvider<T extends SupportedContextItem>(_provider: ContextProvider<T>) { + throw new Error(`Should not be call. Use ILanguageContextProviderService`); + } + + unregisterContextProvider(_providerId: string) { + throw new Error(`Should not be call. Use ILanguageContextProviderService`); + } + + get providers(): ContextProvider<SupportedContextItem>[] { + return this.registryService.getAllProviders([ProviderTarget.Completions]).slice() as ContextProvider<SupportedContextItem>[]; + } + + /** + * Resolves all context providers for the given context. + * Items returned will need to be filtered by schema. + */ + async resolveAllProviders( + completionId: string, + opportunityId: string, + documentContext: DocumentContext, + telemetryData: TelemetryWithExp, + completionCancellationToken?: CancellationToken, + data?: unknown + ): Promise<ResolvedContextItem[]> { + if (completionCancellationToken?.isCancellationRequested) { + logger.debug(this.logTarget, `Resolving context providers cancelled`); + return []; + } + // Pass experiments here if needed. + const activeExperiments: ActiveExperiments = new Map(); + this.instantiationService.invokeFunction(fillInCSharpActiveExperiments, activeExperiments, telemetryData); + const resolvedContextItems: ResolvedContextItem[] = []; + + const _providers = this.providers; + if (_providers.length === 0) { + return resolvedContextItems; + } + + const providersWithMatchScore = await this.matchProviders(_providers, documentContext, telemetryData); + const matchedProviders = providersWithMatchScore.filter(p => p[1] > 0); + const unmatchedProviders = providersWithMatchScore.filter(p => p[1] <= 0); + + // For the unmatched providers, we still want to create a context item, but with an empty data array. + unmatchedProviders.forEach(([provider, score]) => { + const item: ResolvedContextItem = { + providerId: provider.id, + matchScore: score, + resolution: 'none', + resolutionTimeMs: 0, + data: [], + }; + resolvedContextItems.push(item); + }); + + if (matchedProviders.length === 0) { + return resolvedContextItems; + } + if (completionCancellationToken?.isCancellationRequested) { + logger.debug(this.logTarget, `Resolving context providers cancelled`); + return []; + } + + // Fill in the active experiments for the matched providers. + this.instantiationService.invokeFunction(fillInCppVSCodeActiveExperiments, + matchedProviders.map(p => p[0].id), + activeExperiments, + telemetryData + ); + this.instantiationService.invokeFunction(fillInMultiLanguageActiveExperiments, + matchedProviders.map(p => p[0].id), + activeExperiments, + telemetryData + ); + this.instantiationService.invokeFunction(fillInTsActiveExperiments, + matchedProviders.map(p => p[0].id), + activeExperiments, + telemetryData + ); + + const providerCancellationTokenSource = new CancellationTokenSource(); + if (completionCancellationToken) { + const disposable = completionCancellationToken.onCancellationRequested(_ => { + providerCancellationTokenSource.cancel(); + disposable.dispose(); + }); + } + + // Overriding this config with a value of 0 will create an infinite timeout (useful for debugging) + const timeBudget = + this.runtimeMode.isDebugEnabled() && !this.runtimeMode.isRunningInSimulation() + ? 0 + : this.instantiationService.invokeFunction(getContextProviderTimeBudget, documentContext.languageId, telemetryData); + const timeoutEnd = timeBudget > 0 ? Date.now() + timeBudget : Number.MAX_SAFE_INTEGER; + let timeoutId: TimeoutHandle | undefined; + if (timeBudget > 0) { + timeoutId = setTimeout(() => { + providerCancellationTokenSource.cancel(); + providerCancellationTokenSource.dispose(); + }, timeBudget); + } + + const resolutionMap: Map<string, ResolveResult<SupportedContextItem>> = new Map(); + const request: ResolveRequest = { + completionId, + opportunityId, + documentContext, + activeExperiments, + timeBudget, + timeoutEnd, + data, + }; + for (const [provider] of matchedProviders) { + const stats = this.contextProviderStatistics + .getPreviousStatisticsForCompletion(completionId) + ?.get(provider.id); + + if (stats) { + request.previousUsageStatistics = stats; + } + + const pendingContextItem = provider.resolver.resolve(request, providerCancellationTokenSource.token); + resolutionMap.set(provider.id, pendingContextItem); + } + + const statistics = this.contextProviderStatistics.getStatisticsForCompletion(completionId); + statistics.setOpportunityId(opportunityId); + + const results = await resolveAll(resolutionMap, providerCancellationTokenSource.token); + + // Once done, clear the timeout so that we don't cancel the request once it has finished. + if (timeoutId) { + clearTimeout(timeoutId); + } + + for (const [provider, score] of matchedProviders) { + const result = results.get(provider.id); + if (result) { + if (result.status === 'error') { + if (!isCancellationError(result.reason)) { + logger.error(this.logTarget, `Error resolving context from ${provider.id}: `, result.reason); + } + resolvedContextItems.push({ + providerId: provider.id, + matchScore: score, + resolution: result.status, + resolutionTimeMs: result.resolutionTime, + data: [], + }); + } else { + const mergedItems: SupportedContextItem[] = [...(result.value ?? [])]; + if (result.status === 'none' || result.status === 'partial') { + logger.info(this.logTarget, `Context provider ${provider.id} exceeded time budget of ${timeBudget}ms`); + if (provider.resolver.resolveOnTimeout) { + try { + const fallbackItems = provider.resolver.resolveOnTimeout(request); + + if (isArrayOfT(fallbackItems)) { + mergedItems.push(...fallbackItems); + } else if (fallbackItems) { + mergedItems.push(fallbackItems); + } + + if (mergedItems.length > 0) { + result.status = 'partial'; + } + } catch (error) { + logger.error(this.logTarget, `Error in fallback logic for context provider ${provider.id}: `, error); + } + } + } + const [supportedItems, invalidItems] = filterSupportedContextItems(mergedItems); + if (invalidItems) { + logger.error(this.logTarget, `Dropped ${invalidItems} context items from ${provider.id} due to invalid schema`); + } + const filteredItemsWithId = this.instantiationService.invokeFunction(addOrValidateContextItemsIDs, supportedItems); + + const resolvedContextItem: ResolvedContextItem = { + providerId: provider.id, + matchScore: score, + resolution: result.status, + resolutionTimeMs: result.resolutionTime, + data: filteredItemsWithId, + }; + + resolvedContextItems.push(resolvedContextItem); + } + statistics.setLastResolution(provider.id, result.status); + } else { + // This can't happen + logger.error(this.logTarget, `Context provider ${provider.id} not found in results`); + } + } + // Sort the results by match score, so that the highest match score is first. + return resolvedContextItems.sort((a, b) => b.matchScore - a.matchScore); + } + + private async matchProviders( + providers: ContextProvider<SupportedContextItem>[], + documentContext: DocumentContext, + telemetryData: TelemetryWithExp + ): Promise<[ContextProvider<SupportedContextItem>, number][]> { + const activeContextProviders = this.instantiationService.invokeFunction(getActiveContextProviders, documentContext.languageId, telemetryData); + const enableAllProviders = activeContextProviders.length === 1 && activeContextProviders[0] === '*'; + + const providersWithScore = await Promise.all( + providers.map(async provider => { + if (!enableAllProviders && !activeContextProviders.includes(provider.id)) { + return [provider, 0] as [ContextProvider<SupportedContextItem>, number]; + } + + const matchScore = await this.match(this.instantiationService, provider.selector, documentContext); + return [provider, matchScore] as [ContextProvider<SupportedContextItem>, number]; + }) + ); + return providersWithScore; + } +} + +export class MutableContextProviderRegistry extends CoreContextProviderRegistry { + + private _providers: ContextProvider<SupportedContextItem>[] = []; + + constructor( + match: ContextProviderMatchFunction, + @ILanguageContextProviderService registryService: ILanguageContextProviderService, + @ICompletionsRuntimeModeService runtimeMode: ICompletionsRuntimeModeService, + @IInstantiationService instantiationService: IInstantiationService, + @ICompletionsLogTargetService logTarget: ICompletionsLogTargetService, + @ICompletionsContextProviderService contextProviderStatistics: ICompletionsContextProviderService, + ) { + super(match, registryService, runtimeMode, instantiationService, logTarget, contextProviderStatistics); + } + + override registerContextProvider<T extends SupportedContextItem>(provider: ContextProvider<T>) { + if (provider.id.includes(',') || provider.id.includes('*')) { + throw new Error( + `A context provider id cannot contain a comma or an asterisk. The id ${provider.id} is invalid.` + ); + } + if (this._providers.find(p => p.id === provider.id)) { + throw new Error(`A context provider with id ${provider.id} has already been registered`); + } + this._providers.push(provider); + } + + override unregisterContextProvider(providerId: string) { + this._providers = this._providers.filter(p => p.id !== providerId); + } + + override get providers() { + return this._providers.slice().concat(super.providers); + } +} + +export class CachedContextProviderRegistry implements ICompletionsContextProviderRegistryService { + declare _serviceBrand: undefined; + // We don't need to cache many items, since initially we will only hold the cache for + // the duration of a single completion request. + private _cachedContextItems: LRUCacheMap<string, ResolvedContextItem[]> = new LRUCacheMap(5); + + private readonly delegate: CoreContextProviderRegistry; + + constructor( + registry: new (match: ContextProviderMatchFunction) => CoreContextProviderRegistry, + match: ContextProviderMatchFunction, + @IInstantiationService instantiationService: IInstantiationService, + ) { + this.delegate = instantiationService.createInstance(registry, match); + } + + registerContextProvider<T extends SupportedContextItem>(provider: ContextProvider<T>): void { + this.delegate.registerContextProvider(provider); + } + + unregisterContextProvider(providerId: string): void { + this.delegate.unregisterContextProvider(providerId); + } + + get providers(): ContextProvider<SupportedContextItem>[] { + return this.delegate.providers; + } + + async resolveAllProviders( + completionId: string, + opportunityId: string, + documentContext: DocumentContext, + telemetryData: TelemetryWithExp, + completionToken?: CancellationToken, + data?: unknown + ): Promise<ResolvedContextItem[]> { + const cachedItems = this._cachedContextItems.get(completionId); + + if (completionId && cachedItems && cachedItems.length > 0) { + return cachedItems; + } + + const resolvedContextItems = await this.delegate.resolveAllProviders( + completionId, + opportunityId, + documentContext, + telemetryData, + completionToken, + data + ); + + if (resolvedContextItems.length > 0 && completionId) { + this._cachedContextItems.set(completionId, resolvedContextItems); + } + + return resolvedContextItems; + } +} + +export function telemetrizeContextItems( + contextProvider: ICompletionsContextProviderService, + completionId: string, + resolvedContextItems: ResolvedContextItem[] +) { + const contextProviderStatistics = contextProvider.getStatisticsForCompletion(completionId); + const contextProviderTelemetry: ContextProviderTelemetry[] = resolvedContextItems.map(p => { + const { providerId, resolution, resolutionTimeMs, matchScore, data } = p; + + const providerStatistics = contextProviderStatistics.get(providerId); + let usage = providerStatistics?.usage ?? 'none'; + + // Unmatched providers are special: we still want to telemetrize them, but we don't + // rely on the statistics since those will refer to the last time it was matched! + if (matchScore <= 0 || resolution === 'none' || resolution === 'error') { + usage = 'none'; + } + + const contextProviderTelemetry: ContextProviderTelemetry = { + providerId, + resolution, + resolutionTimeMs, + usage, + usageDetails: providerStatistics?.usageDetails, + matched: matchScore > 0, + numResolvedItems: data.length, + }; + + const numUsedItems = + providerStatistics?.usageDetails !== undefined + ? providerStatistics?.usageDetails.filter( + i => i.usage === 'full' || i.usage === 'partial' || i.usage === 'partial_content_excluded' + ).length + : undefined; + + const numPartiallyUsedItems = + providerStatistics?.usageDetails !== undefined + ? providerStatistics?.usageDetails.filter( + i => i.usage === 'partial' || i.usage === 'partial_content_excluded' + ).length + : undefined; + + // TODO: Inline this above once promptlib has been removed + if (numUsedItems !== undefined) { + contextProviderTelemetry.numUsedItems = numUsedItems; + } + if (numPartiallyUsedItems !== undefined) { + contextProviderTelemetry.numPartiallyUsedItems = numPartiallyUsedItems; + } + + return contextProviderTelemetry; + }); + + return contextProviderTelemetry; +} + +export function matchContextItems(resolvedContextItem: ResolvedContextItem): boolean { + return resolvedContextItem.matchScore > 0 && resolvedContextItem.resolution !== 'error'; +} + +function getActiveContextProviders(accessor: ServicesAccessor, languageId: string, telemetryData: TelemetryWithExp): string[] { + const expContextProviders = getExpContextProviders(accessor, languageId, telemetryData); + const configContextProviders: string[] = getConfig(accessor, ConfigKey.ContextProviders) ?? []; + + if ( + (expContextProviders.length === 1 && expContextProviders[0] === '*') || + (configContextProviders.length === 1 && configContextProviders[0] === '*') + ) { + return ['*']; + } + + // Merge the two arrays and deduplicate + const defaultContextProviders = accessor.get(ICompletionsDefaultContextProviders).getIds(); + return Array.from(new Set([...defaultContextProviders, ...expContextProviders, ...configContextProviders])); +} + +/** + * This only returns the context providers that are enabled by EXP. + * Use `getActiveContextProviders` to get the context providers that are enabled by both EXP and config. + */ +function getExpContextProviders(accessor: ServicesAccessor, languageId: string, telemetryData: TelemetryWithExp): string[] { + if (accessor.get(ICompletionsRuntimeModeService).isDebugEnabled()) { + return ['*']; + } + const featuresService = accessor.get(ICompletionsFeaturesService); + const result = featuresService.contextProviders(telemetryData); + const langSpecific = featuresService.getContextProviderExpSettings(languageId); + if (langSpecific !== undefined) { + for (const id of langSpecific.ids) { + if (!result.includes(id)) { + result.push(id); + } + } + } + return result; +} + +export function useContextProviderAPI(accessor: ServicesAccessor, languageId: string, telemetryData: TelemetryWithExp) { + return getActiveContextProviders(accessor, languageId, telemetryData).length > 0; +} + +function getContextProviderTimeBudget(accessor: ServicesAccessor, languageId: string, telemetryData: TelemetryWithExp): number { + const configTimeout = getConfig<number | undefined>(accessor, ConfigKey.ContextProviderTimeBudget); + if (configTimeout !== undefined && typeof configTimeout === 'number') { + return configTimeout; + } + + return accessor.get(ICompletionsFeaturesService).contextProviderTimeBudget(languageId, telemetryData); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryCSharp.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryCSharp.ts new file mode 100644 index 0000000..bfd0473 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryCSharp.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { ICompletionsLogTargetService, logger } from '../logger'; +import { TelemetryWithExp } from '../telemetry'; +import { ActiveExperiments } from './contextProviderRegistry'; + +interface ContextProviderParams { + [key: string]: string | number | boolean; +} + +export function fillInCSharpActiveExperiments( + accessor: ServicesAccessor, + activeExperiments: ActiveExperiments, + telemetryData: TelemetryWithExp +): boolean { + const featuresService = accessor.get(ICompletionsFeaturesService); + const logTarget = accessor.get(ICompletionsLogTargetService); + try { + const csharpContextProviderParams = featuresService.csharpContextProviderParams(telemetryData); + if (csharpContextProviderParams) { + const params = JSON.parse(csharpContextProviderParams) as ContextProviderParams; + for (const [key, value] of Object.entries(params)) { activeExperiments.set(key, value); } + } else { + const params = featuresService.getContextProviderExpSettings('csharp')?.params; + if (params) { + for (const [key, value] of Object.entries(params)) { activeExperiments.set(key, value); } + } + } + } catch (e) { + logger.debug(logTarget, `Failed to get the active C# experiments for the Context Provider API`, e); + return false; + } + return true; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryCpp.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryCpp.ts new file mode 100644 index 0000000..83701fc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryCpp.ts @@ -0,0 +1,62 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { ICompletionsLogTargetService, logger } from '../logger'; +import { TelemetryWithExp } from '../telemetry'; +import { ActiveExperiments } from './contextProviderRegistry'; + +interface CppContextProviderParams { + [key: string]: string | number | boolean; +} + +const cppContextProviderParamsDefault: CppContextProviderParams = { + maxSnippetLength: 3000, + maxSnippetCount: 7, + enabledFeatures: 'Deferred', + timeBudgetMs: 7, + doAggregateSnippets: true, +}; + +const VSCodeCppContextProviderId = 'ms-vscode.cpptools'; + +export function fillInCppVSCodeActiveExperiments( + accessor: ServicesAccessor, + matchedContextProviders: string[], + activeExperiments: ActiveExperiments, + telemetryData: TelemetryWithExp +): void { + if ( + (matchedContextProviders.length === 1 && matchedContextProviders[0] === '*') || + matchedContextProviders.includes(VSCodeCppContextProviderId) + ) { + addActiveExperiments(accessor, activeExperiments, telemetryData); + } +} + +function addActiveExperiments(accessor: ServicesAccessor, activeExperiments: ActiveExperiments, telemetryData: TelemetryWithExp) { + try { + const featuresService = accessor.get(ICompletionsFeaturesService); + const logTarget = accessor.get(ICompletionsLogTargetService); + let params = cppContextProviderParamsDefault; + const cppContextProviderParams = featuresService.cppContextProviderParams(telemetryData); + if (cppContextProviderParams) { + try { + params = JSON.parse(cppContextProviderParams) as CppContextProviderParams; + } catch (e) { + logger.error(logTarget, 'Failed to parse cppContextProviderParams', e); + } + } else { + const langSpecific = featuresService.getContextProviderExpSettings('cpp')?.params; + if (langSpecific) { + params = { ...langSpecific }; + } + } + for (const [key, value] of Object.entries(params)) { activeExperiments.set(key, value); } + } catch (e) { + logger.exception(accessor, e, 'fillInCppActiveExperiments'); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts new file mode 100644 index 0000000..7ff4ee9 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts @@ -0,0 +1,107 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { ICompletionsLogTargetService, logger } from '../logger'; +import { TelemetryWithExp } from '../telemetry'; +import { ActiveExperiments } from './contextProviderRegistry'; + +const MULTI_LANGUAGE_CONTEXT_PROVIDER_ID = 'fallbackContextProvider'; + +/** + * Parameters for configuring the multi-language context provider. + */ +interface MultiLanguageContextProviderParams { + /** + * The maximum number of context items to include in the multi-language context. + * This controls the number of relevant context entries that can be retrieved + * and processed by the provider. + */ + mlcpMaxContextItems: number; + + /** + * The maximum number of symbol matches to include in the multi-language context. + * This determines the upper limit on the number of symbol-based matches that + * can be considered by the provider. + */ + mlcpMaxSymbolMatches: number; + + /** + * Enable imports in the multi-language context provider. + * If set to true, the provider will include import statements in the context. + */ + mlcpEnableImports: boolean; +} + +export const multiLanguageContextProviderParamsDefault: MultiLanguageContextProviderParams = { + mlcpMaxContextItems: 20, + mlcpMaxSymbolMatches: 20, + mlcpEnableImports: false, +}; + +export function fillInMultiLanguageActiveExperiments( + accessor: ServicesAccessor, + matchedContextProviders: string[], + activeExperiments: ActiveExperiments, + telemetryData: TelemetryWithExp +): void { + if ( + (matchedContextProviders.length === 1 && matchedContextProviders[0] === '*') || + matchedContextProviders.includes(MULTI_LANGUAGE_CONTEXT_PROVIDER_ID) + ) { + addActiveExperiments(accessor, activeExperiments, telemetryData); + } +} + +function addActiveExperiments(accessor: ServicesAccessor, activeExperiments: ActiveExperiments, telemetryData: TelemetryWithExp) { + try { + const params = getMultiLanguageContextProviderParamsFromExp(accessor, telemetryData); + for (const [key, value] of Object.entries(params)) { activeExperiments.set(key, value as number); } + } catch (e) { + logger.exception(accessor, e, 'fillInMultiLanguageActiveExperiments'); + } +} + +function getMultiLanguageContextProviderParamsFromExp( + accessor: ServicesAccessor, + telemetryData: TelemetryWithExp +): MultiLanguageContextProviderParams { + let params = multiLanguageContextProviderParamsDefault; + + const logTarget = accessor.get(ICompletionsLogTargetService); + const featuresService = accessor.get(ICompletionsFeaturesService); + const multiLanguageContextProviderParams = featuresService.multiLanguageContextProviderParams(telemetryData); + + if (multiLanguageContextProviderParams) { + try { + params = JSON.parse(multiLanguageContextProviderParams) as MultiLanguageContextProviderParams; + } catch (e) { + logger.error(logTarget, 'Failed to parse multiLanguageContextProviderParams', e); + } + } + + return params; +} + +export function getMultiLanguageContextProviderParamsFromActiveExperiments( + activeExperiments: Map<string, string | number | boolean | string[]> +): MultiLanguageContextProviderParams { + const params = { ...multiLanguageContextProviderParamsDefault }; + + if (activeExperiments.has('mlcpMaxContextItems')) { + params.mlcpMaxContextItems = Number(activeExperiments.get('mlcpMaxContextItems')); + } + + if (activeExperiments.has('mlcpMaxSymbolMatches')) { + params.mlcpMaxSymbolMatches = Number(activeExperiments.get('mlcpMaxSymbolMatches')); + } + + if (activeExperiments.has('mlcpEnableImports')) { + params.mlcpEnableImports = String(activeExperiments.get('mlcpEnableImports')) === 'true'; + } + + return params; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts new file mode 100644 index 0000000..8fd86ef --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts @@ -0,0 +1,50 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { ICompletionsLogTargetService, logger } from '../logger'; +import { TelemetryWithExp } from '../telemetry'; +import { ActiveExperiments } from './contextProviderRegistry'; + +export const TS_CONTEXT_PROVIDER_ID = 'typescript-ai-context-provider'; + +interface ContextProviderParams { + [key: string]: string | number | boolean; +} + +export function fillInTsActiveExperiments( + accessor: ServicesAccessor, + matchedContextProviders: string[], + activeExperiments: ActiveExperiments, + telemetryData: TelemetryWithExp +): boolean { + if ( + !( + (matchedContextProviders.length === 1 && matchedContextProviders[0] === '*') || + matchedContextProviders.includes(TS_CONTEXT_PROVIDER_ID) + ) + ) { + return false; + } + const logTarget = accessor.get(ICompletionsLogTargetService); + const featuresService = accessor.get(ICompletionsFeaturesService); + try { + const tsContextProviderParams = featuresService.tsContextProviderParams(telemetryData); + if (tsContextProviderParams) { + const params = JSON.parse(tsContextProviderParams) as ContextProviderParams; + for (const [key, value] of Object.entries(params)) { activeExperiments.set(key, value); } + } else { + const params = featuresService.getContextProviderExpSettings('typescript')?.params; + if (params) { + for (const [key, value] of Object.entries(params)) { activeExperiments.set(key, value); } + } + } + } catch (e) { + logger.debug(logTarget, `Failed to get the active TypeScript experiments for the Context Provider API`, e); + return false; + } + return true; +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviderStatistics.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderStatistics.ts new file mode 100644 index 0000000..dc59d5f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviderStatistics.ts @@ -0,0 +1,211 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { ComponentStatistics } from '../../../prompt/src/components/components'; +import { + ContextItemOrigin, + ContextItemUsageDetails, + ContextUsageStatistics, + ResolutionStatus, + SupportedContextItemType, + UsageStatus, +} from '../../../types/src'; +import { LRUCacheMap } from '../helpers/cache'; +import { SupportedContextItemWithId } from './contextProviders/contextItemSchemas'; + +export type PromptExpectation = 'included' | 'content_excluded'; + +export type PromptMatcher = { + source: SupportedContextItemWithId; + expectedTokens: number; + actualTokens: number; +}; + +export const ICompletionsContextProviderService = createServiceIdentifier<ICompletionsContextProviderService>('ICompletionsContextProviderService'); +export interface ICompletionsContextProviderService { + readonly _serviceBrand: undefined; + + getStatisticsForCompletion(completionId: string): PerCompletionContextProviderStatistics; + getPreviousStatisticsForCompletion(completionId: string): PerCompletionContextProviderStatistics | undefined; +} + +export class ContextProviderStatistics implements ICompletionsContextProviderService { + declare _serviceBrand: undefined; + + private statistics = new LRUCacheMap<string, PerCompletionContextProviderStatistics>(25); + + constructor( + private readonly createStatistics: () => PerCompletionContextProviderStatistics = () => + new PerCompletionContextProviderStatistics() + ) { } + + getStatisticsForCompletion(completionId: string): PerCompletionContextProviderStatistics { + const statistics = this.statistics.get(completionId); + if (statistics) { + return statistics; + } + const newStatistics = this.createStatistics(); + this.statistics.set(completionId, newStatistics); + return newStatistics; + } + + getPreviousStatisticsForCompletion(completionId: string) { + const keys = Array.from(this.statistics.keys()); + for (let i = keys.length - 1; i >= 0; i--) { + const key = keys[i]; + if (key !== completionId) { + return this.statistics.peek(key); + } + } + return undefined; + } +} + +export class PerCompletionContextProviderStatistics { + + public opportunityId: string | undefined; + + // Keyed by the providerId, contains an array of tuples [context item, expectation] + protected _expectations = new Map<string, [SupportedContextItemWithId, PromptExpectation][]>(); + protected _lastResolution = new Map<string, ResolutionStatus>(); + protected _statistics = new Map<string, ContextUsageStatistics>(); + + constructor() { + this.opportunityId = undefined; + } + + addExpectations(providerId: string, expectations: [SupportedContextItemWithId, PromptExpectation][]) { + const providerExpectations = this._expectations.get(providerId) ?? []; + this._expectations.set(providerId, [...providerExpectations, ...expectations]); + } + + clearExpectations() { + this._expectations.clear(); + } + + setLastResolution(providerId: string, resolution: ResolutionStatus) { + this._lastResolution.set(providerId, resolution); + } + + setOpportunityId(opportunityId: string) { + this.opportunityId = opportunityId; + } + + get(providerId: string): ContextUsageStatistics | undefined { + return this._statistics.get(providerId); + } + + getAllUsageStatistics(): IterableIterator<[string, ContextUsageStatistics]> { + return this._statistics.entries(); + } + + computeMatch(promptMatchers: PromptMatcher[]) { + try { + for (const [providerId, expectations] of this._expectations) { + if (expectations.length === 0) { + continue; + } + + const resolution = this._lastResolution.get(providerId) ?? 'none'; + if (resolution === 'none' || resolution === 'error') { + this._statistics.set(providerId, { + usage: 'none', + resolution, + }); + continue; + } + + const providerUsageDetails: ContextItemUsageDetails[] = []; + + for (const [item, expectation] of expectations) { + const itemDetails: { + id: string; + type: SupportedContextItemType; + origin?: ContextItemOrigin; + } = { + id: item.id, + type: item.type, + }; + + if (item.origin) { + itemDetails.origin = item.origin; + } + + if (expectation === 'content_excluded') { + providerUsageDetails.push({ + ...itemDetails, + usage: 'none_content_excluded', + }); + continue; + } + + const itemStatistics = promptMatchers.find(component => component.source === item); + + if (itemStatistics === undefined) { + providerUsageDetails.push({ + ...itemDetails, + // In this case, the item didn't make to elision, despite being expected. + usage: 'error', + }); + } else { + providerUsageDetails.push({ + ...itemDetails, + usage: + itemStatistics.expectedTokens > 0 && + itemStatistics.expectedTokens === itemStatistics.actualTokens + ? 'full' + : itemStatistics.actualTokens > 0 + ? 'partial' + : 'none', + expectedTokens: itemStatistics.expectedTokens, + actualTokens: itemStatistics.actualTokens, + }); + } + } + + const usedItems = providerUsageDetails.reduce((acc, item) => { + if (item.usage === 'full') { + return acc + 1; + } else if (item.usage === 'partial') { + return acc + 0.5; + } + return acc; + }, 0); + const usedPercentage = usedItems / expectations.length; + const usage: UsageStatus = usedPercentage === 1 ? 'full' : usedPercentage === 0 ? 'none' : 'partial'; + this._statistics.set(providerId, { + resolution, + usage, + usageDetails: providerUsageDetails, + }); + } + } finally { + // Remove expectations and resolutions no matter what happens + this.clearExpectations(); + this._lastResolution.clear(); + } + } +} + +export function componentStatisticsToPromptMatcher(promptComponentStatistics: ComponentStatistics[]): PromptMatcher[] { + return promptComponentStatistics + .map(component => { + if ( + component.source === undefined || + component.expectedTokens === undefined || + component.actualTokens === undefined + ) { + return; + } + + return { + source: component.source as SupportedContextItemWithId, + expectedTokens: component.expectedTokens, + actualTokens: component.actualTokens, + }; + }) + .filter(p => p !== undefined); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/codeSnippets.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/codeSnippets.ts new file mode 100644 index 0000000..afa7e38 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/codeSnippets.ts @@ -0,0 +1,83 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TextDocumentValidation } from '../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { ResolvedContextItem } from '../contextProviderRegistry'; +import { ICompletionsContextProviderService, PromptExpectation } from '../contextProviderStatistics'; +import { CodeSnippetWithId, filterContextItemsByType } from './contextItemSchemas'; + +const CONTENT_EXCLUDED_EXPECTATION: PromptExpectation = 'content_excluded'; + +type SnippetWithProviderInfo = { + providerId: string; + data: CodeSnippetWithId; +}; + +export async function getCodeSnippetsFromContextItems( + accessor: ServicesAccessor, + completionId: string, + resolvedContextItems: ResolvedContextItem[], + languageId: string +): Promise<CodeSnippetWithId[]> { + const codeSnippetContextItems = filterContextItemsByType(resolvedContextItems, 'CodeSnippet'); + + if (codeSnippetContextItems.length === 0) { + return []; + } + + // Expand snippets and collect URIs + const allUris = new Set<string>(); + const mappedSnippets: SnippetWithProviderInfo[] = codeSnippetContextItems.flatMap(item => + item.data.map(data => { + allUris.add(data.uri); + data.additionalUris?.forEach(uri => allUris.add(uri)); + return { providerId: item.providerId, data }; + }) + ); + + // Validate all URIs at once: we already know they are distinct + const contextProviderStatistics = accessor.get(ICompletionsContextProviderService); + const tdm = accessor.get(ICompletionsTextDocumentManagerService); + const validationMap = new Map<string, TextDocumentValidation>(); + await Promise.all( + Array.from(allUris).map(async uri => { + validationMap.set(uri, await tdm.getTextDocumentValidation({ uri })); + }) + ); + + // Process only valid snippets + const statistics = contextProviderStatistics.getStatisticsForCompletion(completionId); + return mappedSnippets + .filter(snippet => { + const urisToCheck = [snippet.data.uri, ...(snippet.data.additionalUris ?? [])]; + const isValid = urisToCheck.every(uri => validationMap.get(uri)?.status === 'valid'); + + // Set expectations regardless of validity + if (isValid) { + statistics.addExpectations(snippet.providerId, [[snippet.data, 'included']]); + } else { + statistics.addExpectations(snippet.providerId, [[snippet.data, CONTENT_EXCLUDED_EXPECTATION]]); + } + + return isValid; + }) + .map(snippet => snippet.data); +} + +export type CodeSnippetWithRelativePath = { snippet: CodeSnippetWithId; relativePath?: string }; + +export function addRelativePathToCodeSnippets( + tdm: ICompletionsTextDocumentManagerService, + codeSnippets: CodeSnippetWithId[] +): CodeSnippetWithRelativePath[] { + return codeSnippets.map(codeSnippet => { + return { + snippet: codeSnippet, + relativePath: tdm.getRelativePath(codeSnippet), + }; + }); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/contextItemSchemas.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/contextItemSchemas.ts new file mode 100644 index 0000000..e4821bc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/contextItemSchemas.ts @@ -0,0 +1,201 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Diagnostic } from 'vscode'; +import { URI } from '../../../../../../../util/vs/base/common/uri'; +import { generateUuid } from '../../../../../../../util/vs/base/common/uuid'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + CodeSnippet, + ContextItemOrigin, + DiagnosticBag, + SupportedContextItem, + SupportedContextItemType, + Trait, +} from '../../../../types/src'; +import { ICompletionsLogTargetService, logger } from '../../logger'; +import { ResolvedContextItem } from '../contextProviderRegistry'; + +namespace ContextItemSchema { + export function is(item: SupportedContextItem): boolean { + if (item.importance !== undefined) { + if (typeof item.importance !== 'number' || !Number.isInteger(item.importance) || item.importance < 0 || item.importance > 100) { + return false; + } + } + if (item.id !== undefined) { + if (typeof item.id !== 'string') { + return false; + } + } + if (item.origin !== undefined) { + if (!ContextItemOrigin.is(item.origin)) { + return false; + } + } + return true; + } +} + +namespace TraitSchema { + export function is(item: SupportedContextItem): item is Trait { + if (!ContextItemSchema.is(item)) { + return false; + } + const candidate = item as Trait; + return typeof candidate.name === 'string' && typeof candidate.value === 'string'; + } +} + +namespace CodeSnippetSchema { + export function is(item: SupportedContextItem): item is CodeSnippet { + if (!ContextItemSchema.is(item)) { + return false; + } + const candidate = item as CodeSnippet; + if (typeof candidate.uri !== 'string' || typeof candidate.value !== 'string') { + return false; + } + if (candidate.additionalUris === undefined) { + return true; + } + if (!Array.isArray(candidate.additionalUris)) { + return false; + } + for (const uri of candidate.additionalUris) { + if (typeof uri !== 'string') { + return false; + } + } + return true; + } +} + +namespace DiagnosticBagSchema { + export function is(item: SupportedContextItem): item is DiagnosticBag { + if (!ContextItemSchema.is(item)) { + return false; + } + const candidate = item as DiagnosticBag; + if (!(URI.isUri(candidate.uri))) { + return false; + } + if (!Array.isArray(candidate.values)) { + return false; + } + for (const diagnostic of candidate.values) { + if (!(diagnostic instanceof Diagnostic)) { + return false; + } + } + return true; + } +} + +namespace SupportedContextItemSchema { + export function is(item: SupportedContextItem): SupportedContextItemType | undefined { + if (TraitSchema.is(item)) { + return 'Trait'; + } else if (CodeSnippetSchema.is(item)) { + return 'CodeSnippet'; + } else if (DiagnosticBagSchema.is(item)) { + return 'DiagnosticBag'; + } + return undefined; + } +} + +/** + * + * Internal types and validation functions for context items + */ + +/** + * Construct the final types, which may include required properties that the base types do not have. + */ + +export type TraitWithId = Trait & { id: string; type: 'Trait' }; +export type CodeSnippetWithId = CodeSnippet & { id: string; type: 'CodeSnippet' }; +export type DiagnosticBagWithId = DiagnosticBag & { id: string; type: 'DiagnosticBag' }; +export type SupportedContextItemWithId = TraitWithId | CodeSnippetWithId | DiagnosticBagWithId; + +export function filterContextItemsByType<S extends SupportedContextItemType>( + resolvedContextItems: ResolvedContextItem[], + type: S +): ResolvedContextItem<Extract<SupportedContextItemWithId, { type: S }>>[] { + return resolvedContextItems + .map(item => { + const filteredData = item.data.filter(data => data.type === type) as Extract< + SupportedContextItemWithId, + { type: S } + >[]; + + return filteredData.length > 0 ? { ...item, data: filteredData } : undefined; + }) + .filter(r => r !== undefined) as ResolvedContextItem<Extract<SupportedContextItemWithId, { type: S }>>[]; +} + +type SupportedContextItemWithType = SupportedContextItem & { type: SupportedContextItemType }; + +export function filterSupportedContextItems( + contextItems: SupportedContextItem[] +): [SupportedContextItemWithType[], number] { + const filteredItems: SupportedContextItemWithType[] = []; + let invalidItemsCounter = 0; + + contextItems.forEach(item => { + const type = SupportedContextItemSchema.is(item); + if (type !== undefined) { + filteredItems.push({ + ...item, + type, + }); + } else { + invalidItemsCounter++; + } + }); + + return [filteredItems, invalidItemsCounter]; +} + +/** + * + * Only allow alphanumeric characters and hyphens to remove symbols that could + * be problematic when used as prompt components keys. + */ +function validateContextItemId(id: string): boolean { + return id.length > 0 && id.replaceAll(/[^a-zA-Z0-9-]/g, '').length === id.length; +} + +/** + * Assigns a random ID if it wasn't assigned by the context provider. + * Invalid or duplicate IDs are replaced with valid ones and logged to avoid dropping the context + * and worsen the user experience. + */ +export function addOrValidateContextItemsIDs( + accessor: ServicesAccessor, + contextItems: SupportedContextItemWithType[] +): SupportedContextItemWithId[] { + const seenIds = new Set<string>(); + const logTarget = accessor.get(ICompletionsLogTargetService); + + const contextItemsWithId: SupportedContextItemWithId[] = []; + for (const item of contextItems) { + let id = item.id ?? generateUuid(); + if (!validateContextItemId(id)) { + const newID = generateUuid(); + logger.error(logTarget, `Invalid context item ID ${id}, replacing with ${newID}`); + id = newID; + } + if (seenIds.has(id)) { + const newID = generateUuid(); + logger.error(logTarget, `Duplicate context item ID ${id}, replacing with ${newID}`); + id = newID; + } + seenIds.add(id); + contextItemsWithId.push({ ...item, id } as SupportedContextItemWithId); + } + return contextItemsWithId; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/diagnostics.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/diagnostics.ts new file mode 100644 index 0000000..66c6000 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/diagnostics.ts @@ -0,0 +1,78 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import type { TextDocumentValidation } from '../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { ResolvedContextItem } from '../contextProviderRegistry'; +import { ICompletionsContextProviderService, type PromptExpectation } from '../contextProviderStatistics'; +import { filterContextItemsByType, type DiagnosticBagWithId } from './contextItemSchemas'; + +const CONTENT_EXCLUDED_EXPECTATION: PromptExpectation = 'content_excluded'; + +export async function getDiagnosticsFromContextItems( + accessor: ServicesAccessor, + completionId: string, + resolvedContextItems: ResolvedContextItem[] +): Promise<DiagnosticBagWithId[]> { + const diagnosticBags = filterContextItemsByType(resolvedContextItems, 'DiagnosticBag'); + + // Set expectations for the diagnostics provided. + for (const item of diagnosticBags) { + setupExpectationsForDiagnostics(accessor, completionId, item.data, item.providerId); + } + + // Flatten and sort the traits by importance. + // TODO: once we deprecate the old API, importance should also dictate elision. + const allUris: Set<string> = new Set(); + const diagnosticBagsWithProviderId: { providerId: string; bag: DiagnosticBagWithId }[] = []; + for (const item of diagnosticBags) { + for (const diagnosticBag of item.data) { + allUris.add(diagnosticBag.uri.toString()); + diagnosticBagsWithProviderId.push({ providerId: item.providerId, bag: diagnosticBag }); + } + } + + if (diagnosticBagsWithProviderId.length === 0) { + return []; + } + + const contextProviderStatistics = accessor.get(ICompletionsContextProviderService); + const tdm = accessor.get(ICompletionsTextDocumentManagerService); + + const validationMap = new Map<string, TextDocumentValidation>(); + await Promise.all( + Array.from(allUris).map(async uri => { + validationMap.set(uri, await tdm.getTextDocumentValidation({ uri })); + }) + ); + + const statistics = contextProviderStatistics.getStatisticsForCompletion(completionId); + const filteredDiagnosticBags = diagnosticBagsWithProviderId + .filter(item => { + const isValid = validationMap.get(item.bag.uri.toString())?.status === 'valid'; + + // Set expectations regardless of validity + if (isValid) { + statistics.addExpectations(item.providerId, [[item.bag, 'included']]); + } else { + statistics.addExpectations(item.providerId, [[item.bag, CONTENT_EXCLUDED_EXPECTATION]]); + } + + return isValid; + }) + .map(item => item.bag); + + + return filteredDiagnosticBags.sort((a, b) => (a.importance ?? 0) - (b.importance ?? 0)); +} + +function setupExpectationsForDiagnostics(accessor: ServicesAccessor, completionId: string, diagnostics: DiagnosticBagWithId[], providerId: string) { + const statistics = accessor.get(ICompletionsContextProviderService).getStatisticsForCompletion(completionId); + + diagnostics.forEach(t => { + statistics.addExpectations(providerId, [[t, 'included']]); + }); +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/codeSnippets.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/codeSnippets.test.ts new file mode 100644 index 0000000..b593f41 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/codeSnippets.test.ts @@ -0,0 +1,259 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import os from 'os'; +import { IIgnoreService } from '../../../../../../../../platform/ignore/common/ignoreService'; +import { TestingServiceCollection } from '../../../../../../../../platform/test/node/services'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFileSystemService } from '../../../fileSystem'; +import { createLibTestingContext } from '../../../test/context'; +import { FakeFileSystem } from '../../../test/filesystem'; +import { MockIgnoreService } from '../../../test/testContentExclusion'; +import { SimpleTestTextDocumentManager, TestTextDocumentManager } from '../../../test/textDocument'; +import { TextDocumentIdentifier, TextDocumentValidation } from '../../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; +import { ResolvedContextItem } from '../../contextProviderRegistry'; +import { ContextProviderStatistics, ICompletionsContextProviderService } from '../../contextProviderStatistics'; +import { TestContextProviderStatistics } from '../../test/contextProviderStatistics'; +import { getCodeSnippetsFromContextItems } from '../codeSnippets'; +import { CodeSnippetWithId } from '../contextItemSchemas'; + +suite('codeSnippetsContextProvider', function () { + let accessor: ServicesAccessor; + let serviceCollection: TestingServiceCollection; + let tdm: TestTextDocumentManager; + let ignoreService: MockIgnoreService; + const resolvedContextItems: ResolvedContextItem<CodeSnippetWithId>[] = [ + { + providerId: 'testCodeSnippetsProvider1', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + { + uri: 'file:///foo.js', + value: 'foovalue', + additionalUris: ['file:///foo2.js'], + id: '1', + type: 'CodeSnippet', + }, + { + uri: 'file:///bar.js', + value: 'barvalue', + id: '2', + type: 'CodeSnippet', + }, + // Multiple snippets for the same file are allowed + { + uri: 'file:///bar.js', + value: 'anotherbarvalue', + id: '3', + type: 'CodeSnippet', + }, + ], + }, + { + providerId: 'testCodeSnippetsProvider2', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + { uri: 'file:///baz.js', value: 'bazvalue', id: '4', type: 'CodeSnippet' }, + { uri: 'file:///maybe.js', value: 'maybevalue', id: '5', type: 'CodeSnippet' }, + ], + }, + ]; + + setup(function () { + serviceCollection = createLibTestingContext(); + serviceCollection.define(IIgnoreService, new MockIgnoreService()); + accessor = serviceCollection.createTestingAccessor(); + + ignoreService = accessor.get(IIgnoreService) as MockIgnoreService; + tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument('file:///foo.js', 'javascript', 'doesntmatter'); + tdm.setTextDocument('file:///bar.js', 'javascript', 'doesntmatter'); + tdm.setTextDocument('file:///baz.js', 'javascript', 'doesntmatter'); + tdm.setTextDocument('file:///foo2.js', 'javascript', 'doesntmatter'); + }); + + test('can get code snippets from context text providers and flattens them', async function () { + const codeSnippets = await getCodeSnippetsFromContextItems( + accessor, + 'COMPLETION_ID', + resolvedContextItems, + 'javascript' + ); + + assert.deepStrictEqual(codeSnippets.length, 5); + assert.deepStrictEqual( + codeSnippets.map(t => t.value), + ['foovalue', 'barvalue', 'anotherbarvalue', 'bazvalue', 'maybevalue'] + ); + }); + + test('set expectations for contextProviderStatistics', async function () { + const statistics = new TestContextProviderStatistics(); + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define(ICompletionsContextProviderService, new ContextProviderStatistics(() => statistics)); + const accessor = serviceCollectionClone.createTestingAccessor(); + + await getCodeSnippetsFromContextItems(accessor, 'COMPLETION_ID', resolvedContextItems, 'javascript'); + + assert.deepStrictEqual(statistics.expectations.size, 2); + + const expectations = statistics.expectations.get('testCodeSnippetsProvider1'); + assert.ok(expectations); + assert.deepStrictEqual(expectations, [ + [ + { + uri: 'file:///foo.js', + value: 'foovalue', + additionalUris: ['file:///foo2.js'], + id: '1', + type: 'CodeSnippet', + }, + 'included', + ], + [{ uri: 'file:///bar.js', value: 'barvalue', id: '2', type: 'CodeSnippet' }, 'included'], + [{ uri: 'file:///bar.js', value: 'anotherbarvalue', id: '3', type: 'CodeSnippet' }, 'included'], + ]); + + const expectations2 = statistics.expectations.get('testCodeSnippetsProvider2'); + assert.ok(expectations2); + assert.deepStrictEqual(expectations2, [ + [{ uri: 'file:///baz.js', value: 'bazvalue', id: '4', type: 'CodeSnippet' }, 'included'], + [{ uri: 'file:///maybe.js', value: 'maybevalue', id: '5', type: 'CodeSnippet' }, 'included'], + ]); + }); + + test('content excluded files are not returned', async function () { + // maybe.js is set but not content excluded + tdm.setTextDocument('file:///maybe.js', 'javascript', 'doesntmatter'); + + const codeSnippets = await getCodeSnippetsFromContextItems( + accessor, + 'COMPLETION_ID', + resolvedContextItems, + 'javascript' + ); + + assert.deepStrictEqual(codeSnippets.length, 5); + assert.ok(codeSnippets.map(t => t.uri).includes('file:///maybe.js')); + + // If it's content excluded, it's not returned + ignoreService.setBlockListUris(['file:///maybe.js']); + const codeSnippetsAfterExclusion = await getCodeSnippetsFromContextItems( + accessor, + 'COMPLETION_ID', + resolvedContextItems, + 'javascript' + ); + + assert.deepStrictEqual(codeSnippetsAfterExclusion.length, 4); + assert.ok(!codeSnippetsAfterExclusion.map(t => t.uri).includes('file:///maybe.js')); + }); + + test('documents can be read from the file system,', async function () { + // The additionalUri for the code snippet is not open, so we create a fake file system + // entry depending on the OS to test the normalization of the URI. + const drive = os.platform() === 'win32' ? 'c:' : ''; + const uriPrefix = os.platform() === 'win32' ? 'file:///c:' : 'file://'; + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define( + ICompletionsFileSystemService, + new FakeFileSystem({ + [`${drive}/fake2.js`]: 'content', + }) + ); + + // Use a SimpleTestTextDocumentManager to read from the FakeFileSystem + const tdm = accessor.get(IInstantiationService).createInstance(SimpleTestTextDocumentManager); + serviceCollectionClone.define(ICompletionsTextDocumentManagerService, tdm); + const accessorClone = serviceCollectionClone.createTestingAccessor(); + + const additionalUri = `${uriPrefix}/fake2.js`; + + // Set the main uri as an open file + const mainUri = `${uriPrefix}/fake.js`; + tdm.setTextDocument(mainUri, 'javascript', 'doesntmatter'); + + const resolvedContextItems: ResolvedContextItem<CodeSnippetWithId>[] = [ + { + providerId: 'testCodeSnippetsProvider1', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + { + uri: mainUri, + value: 'foovalue', + additionalUris: [additionalUri], + id: '1', + type: 'CodeSnippet', + }, + ], + }, + ]; + + const codeSnippets = await getCodeSnippetsFromContextItems( + accessorClone, + 'COMPLETION_ID', + resolvedContextItems, + 'javascript' + ); + + assert.deepStrictEqual(codeSnippets.length, 1); + }); + + test('content exclusion does not check multiple times', async function () { + const serviceCollectionClone = serviceCollection.clone(); + const tdm = accessor.get(IInstantiationService).createInstance(FakeTextDocumentManager); + serviceCollectionClone.define(ICompletionsTextDocumentManagerService, tdm); + const accessorClone = serviceCollectionClone.createTestingAccessor(); + + await getCodeSnippetsFromContextItems(accessorClone, 'COMPLETION_ID', resolvedContextItems, 'javascript'); + const uris = resolvedContextItems.map(t => t.data.flatMap(d => [d.uri, ...(d.additionalUris ?? [])])).flat(); + assert.ok(uris.length > tdm.checkedUris.length); + assert.deepStrictEqual(tdm.checkedUris.length, new Set(tdm.checkedUris).size); + }); + + test('files are not returned if any of their additionalUris are excluded', async function () { + ignoreService.setBlockListUris(['file:///foo2.js']); + const codeSnippets = await getCodeSnippetsFromContextItems( + accessor, + 'COMPLETION_ID', + resolvedContextItems, + 'javascript' + ); + + assert.deepStrictEqual(codeSnippets.length, 4); + assert.ok(!codeSnippets.map(t => t.uri).includes('file:///foo.js')); + }); + + test('documents do not have to be open', async function () { + tdm.setDiskContents('file:///maybe.js', 'doesntmatter'); + + const codeSnippets = await getCodeSnippetsFromContextItems( + accessor, + 'COMPLETION_ID', + resolvedContextItems, + 'javascript' + ); + + assert.deepStrictEqual(codeSnippets.length, 5); + assert.ok(codeSnippets.map(t => t.uri).includes('file:///maybe.js')); + }); +}); + +class FakeTextDocumentManager extends TestTextDocumentManager { + checkedUris: string[] = []; + + override getTextDocumentValidation(docId: TextDocumentIdentifier): Promise<TextDocumentValidation> { + this.checkedUris.push(docId.uri); + return Promise.resolve({ status: 'valid' }); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/contextItemSchemas.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/contextItemSchemas.test.ts new file mode 100644 index 0000000..0108525 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/contextItemSchemas.test.ts @@ -0,0 +1,137 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ResolvedContextItem } from '../../contextProviderRegistry'; +import { + filterContextItemsByType, + filterSupportedContextItems, + SupportedContextItemWithId, + TraitWithId, +} from '../contextItemSchemas'; +import { SupportedContextItem } from '../../../../../types/src'; +import assert from 'assert'; + +suite('contextItemSchemas', function () { + test('can filter homogeneous context item by schema', function () { + const badItem: ResolvedContextItem = { + providerId: 'doesntmatter', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: ['hello' as unknown as TraitWithId], + }; + const goodItem: ResolvedContextItem = { + providerId: 'doesntmatter', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + { name: 'trait1', value: 'value1', id: '1', type: 'Trait' }, + { name: 'trait2', value: 'value2', id: '2', type: 'Trait' }, + ], + }; + + // Since they are homogeneous, it's either all or nothing. + assert.deepStrictEqual(filterContextItemsByType([badItem], 'Trait'), []); + assert.deepStrictEqual(filterContextItemsByType([goodItem], 'Trait'), [goodItem]); + }); + + test('can filter homogeneous context item lists by schema', function () { + const resolvedContextItems: ResolvedContextItem[] = [ + { + providerId: 'doesntmatter', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: ['hello' as unknown as TraitWithId], + }, + { + providerId: 'doesntmatter', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + { name: 'trait1', value: 'value1', id: '1', type: 'Trait' }, + { name: 'trait2', value: 'value2', id: '2', type: 'Trait' }, + ], + }, + ]; + + assert.deepStrictEqual(filterContextItemsByType(resolvedContextItems, 'Trait'), [resolvedContextItems[1]]); + }); + + test('can filter heterogeneous context item schema', function () { + const data: SupportedContextItemWithId[] = [ + { name: 'trait1', value: 'value1', id: '1', type: 'Trait' }, + { uri: 'file:///foo', value: 'filevalue1', id: '2', type: 'CodeSnippet' }, + ]; + const mixedContextItem: ResolvedContextItem = { + providerId: 'doesntmatter', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data, + }; + + const filteredTraits = filterContextItemsByType([mixedContextItem], 'Trait'); + assert.deepStrictEqual(filteredTraits.length, 1); + assert.deepStrictEqual(filteredTraits[0].data, [data[0]]); + + const filteredFileSnippets = filterContextItemsByType([mixedContextItem], 'CodeSnippet'); + assert.deepStrictEqual(filteredFileSnippets.length, 1); + assert.deepStrictEqual(filteredFileSnippets[0].data, [data[1]]); + }); + + test('can filter heterogeneous context item list by schema', function () { + const resolvedContextItems: ResolvedContextItem[] = [ + { + providerId: 'doesntmatter1', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + { name: 'trait1', value: 'value1', id: '1', type: 'Trait' }, + { uri: 'file:///foo', value: 'filevalue1', id: '2', type: 'CodeSnippet' }, + ], + }, + { + providerId: 'doesntmatter2', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [{ name: 'trait2', value: 'value2', id: '3', type: 'Trait' }], + }, + ]; + + const filteredTraits = filterContextItemsByType(resolvedContextItems, 'Trait'); + assert.deepStrictEqual(filteredTraits.length, 2); + assert.deepStrictEqual(filteredTraits[0].data, [{ name: 'trait1', value: 'value1', id: '1', type: 'Trait' }]); + assert.deepStrictEqual(filteredTraits[1].data, [{ name: 'trait2', value: 'value2', id: '3', type: 'Trait' }]); + }); + + test('validates context items schema', function () { + const resolvedContextItems: SupportedContextItem[] = [ + { name: 'trait1', value: 'value1' }, + { uri: 'file:///foo', value: 'filevalue1' }, + ]; + + const [validItems, invalidItems] = filterSupportedContextItems(resolvedContextItems); + assert.deepStrictEqual(invalidItems, 0); + assert.deepStrictEqual(validItems.length, 2); + }); + + test('items can have optional properties', function () { + const resolvedContextItems = [ + { uri: 'file:///foo', value: 'filevaluewithoptionalprop', optionalProp: 'optional' }, + { uri: 'file:///foo', value: 'filevaluewithoutag' }, + ]; + + const [validItems, invalidItems] = filterSupportedContextItems(resolvedContextItems); + assert.deepStrictEqual(invalidItems, 0); + assert.deepStrictEqual(validItems.length, 2); + // Keeps all optional properties + assert.deepStrictEqual((validItems[0] as unknown as { [key: string]: unknown }).optionalProp, 'optional'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/traits.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/traits.test.ts new file mode 100644 index 0000000..9d36b14 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/test/traits.test.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ServicesAccessor } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { createLibTestingContext } from '../../../test/context'; +import { ResolvedContextItem } from '../../contextProviderRegistry'; +import { ContextProviderStatistics, ICompletionsContextProviderService } from '../../contextProviderStatistics'; +import { TestContextProviderStatistics } from '../../test/contextProviderStatistics'; +import { TraitWithId } from './../contextItemSchemas'; +import { getTraitsFromContextItems } from './../traits'; + +suite('traitsContextProvider', function () { + let accessor: ServicesAccessor; + const resolvedContextItems: ResolvedContextItem<TraitWithId>[] = [ + { + providerId: 'testTraitsProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [ + // This trait should be the last in the list, since higher importance + // is closer to the end of the prompt. + { + name: 'trait_from_context_provider1_1', + value: 'value_1', + importance: 10, + id: '1', + type: 'Trait', + }, + { + name: 'trait_from_context_provider1_2', + value: 'value_2', + id: '2', + type: 'Trait', + }, + ], + }, + { + providerId: 'testTraitsProvider2', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 10, + data: [{ name: 'trait_from_context_provider2_1', value: 'value_3', id: '3', type: 'Trait' }], + }, + ]; + + setup(function () { + const serviceCollection = createLibTestingContext(); + serviceCollection.define( + ICompletionsContextProviderService, + new ContextProviderStatistics(() => new TestContextProviderStatistics()) + ); + accessor = serviceCollection.createTestingAccessor(); + }); + + test('can get traits from context text providers and flattens them', function () { + const traits = getTraitsFromContextItems(accessor, 'COMPLETION_ID', resolvedContextItems); + assert.deepStrictEqual(traits.length, 3); + assert.deepStrictEqual( + traits.map(t => t.name), + ['trait_from_context_provider1_2', 'trait_from_context_provider2_1', 'trait_from_context_provider1_1'] + ); + }); + + test('set expectations for contextProviderStatistics', function () { + + getTraitsFromContextItems(accessor, 'COMPLETION_ID', resolvedContextItems); + + const statistics = accessor + .get(ICompletionsContextProviderService) + .getStatisticsForCompletion('COMPLETION_ID') as TestContextProviderStatistics; + // Prompt components expectations + assert.deepStrictEqual(statistics.expectations.size, 2); + const traitExpectations = statistics.expectations.get('testTraitsProvider'); + assert.ok(traitExpectations); + assert.deepStrictEqual(traitExpectations, [ + [ + { id: '1', name: 'trait_from_context_provider1_1', value: 'value_1', importance: 10, type: 'Trait' }, + 'included', + ], + [{ id: '2', name: 'trait_from_context_provider1_2', value: 'value_2', type: 'Trait' }, 'included'], + ]); + const traitExpectations2 = statistics.expectations.get('testTraitsProvider2'); + assert.ok(traitExpectations2); + assert.deepStrictEqual(traitExpectations2, [ + [{ id: '3', name: 'trait_from_context_provider2_1', value: 'value_3', type: 'Trait' }, 'included'], + ]); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/traits.ts b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/traits.ts new file mode 100644 index 0000000..b388102 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/contextProviders/traits.ts @@ -0,0 +1,69 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { Trait } from '../../../../types/src'; +import { telemetry, TelemetryProperties, TelemetryWithExp } from '../../telemetry'; +import { ResolvedContextItem } from '../contextProviderRegistry'; +import { ICompletionsContextProviderService } from '../contextProviderStatistics'; +import { filterContextItemsByType, TraitWithId } from './contextItemSchemas'; + +export function getTraitsFromContextItems( + accessor: ServicesAccessor, + completionId: string, + resolvedContextItems: ResolvedContextItem[] +): TraitWithId[] { + const traitsContextItems = filterContextItemsByType(resolvedContextItems, 'Trait'); + + // Set expectations for the traits + for (const item of traitsContextItems) { + setupExpectationsForTraits(accessor, completionId, item.data, item.providerId); + } + + // Flatten and sort the traits by importance. + // TODO: once we deprecate the old API, importance should also dictate elision. + const traits: TraitWithId[] = traitsContextItems.flatMap(p => p.data); + return traits.sort((a, b) => (a.importance ?? 0) - (b.importance ?? 0)); +} + +function setupExpectationsForTraits(accessor: ServicesAccessor, completionId: string, traits: TraitWithId[], providerId: string) { + const statistics = accessor.get(ICompletionsContextProviderService).getStatisticsForCompletion(completionId); + + traits.forEach(t => { + statistics.addExpectations(providerId, [[t, 'included']]); + }); +} + +// Maintain a list of names for traits we'd like to report in telemetry. +// The key is the trait name, and the value is the corresponding name of the telemetry property as listed in the hydro schema. +const traitNamesForTelemetry: Map<string, string> = new Map([ + ['TargetFrameworks', 'targetFrameworks'], + ['LanguageVersion', 'languageVersion'], +]); + +export function ReportTraitsTelemetry( + accessor: ServicesAccessor, + eventName: string, + traits: Trait[], + detectedLanguageId: string, + clientLanguageId: string, + telemetryData: TelemetryWithExp +) { + if (traits.length > 0) { + const properties: TelemetryProperties = {}; + properties.detectedLanguageId = detectedLanguageId; + properties.languageId = clientLanguageId; + + for (const trait of traits) { + const mappedTraitName = traitNamesForTelemetry.get(trait.name); + if (mappedTraitName) { + properties[mappedTraitName] = trait.value; + } + } + + const telemetryDataExt = telemetryData.extendedBy(properties, {}); + return telemetry(accessor, eventName, telemetryDataExt); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/parseBlock.ts b/completions-sample-code/vscode-node/lib/src/prompt/parseBlock.ts new file mode 100644 index 0000000..a87e1fb --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/parseBlock.ts @@ -0,0 +1,277 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { getNodeStart, isBlockBodyFinished, isEmptyBlockStart } from '../../../prompt/src/parseBlock'; +import { IPosition, LocationFactory, TextDocumentContents } from '../textDocument'; + +export function parsingBlockFinished( + doc: TextDocumentContents, + position: IPosition +): (completion: string) => Promise<number | undefined> { + const prefix = doc.getText(LocationFactory.range(LocationFactory.position(0, 0), position)); + const offset = doc.offsetAt(position); + const languageId = doc.detectedLanguageId; + + return completion => isBlockBodyFinished(languageId, prefix, completion, offset); +} + +export function isEmptyBlockStartUtil(doc: TextDocumentContents, position: IPosition): Promise<boolean> { + return isEmptyBlockStart(doc.detectedLanguageId, doc.getText(), doc.offsetAt(position)); +} + +export async function getNodeStartUtil( + doc: TextDocumentContents, + position: IPosition, + completion: string +): Promise<IPosition | undefined> { + const prefix = doc.getText(LocationFactory.range(LocationFactory.position(0, 0), position)); + const text = prefix + completion; + const offset = await getNodeStart(doc.detectedLanguageId, text, doc.offsetAt(position)); + if (offset) { + return doc.positionAt(offset); + } +} + + +// TODO: This should probably be language specific +const continuations = [ + // Brace control + '\\{', + '\\}', + '\\[', + '\\]', + '\\(', + '\\)', +].concat( + [ + // Separators in a multi-line list + // ",", ";", "\\|", + // Multi-line comments + // None + // Keywords for same-level control flow + 'then', + 'else', + 'elseif', + 'elif', + 'catch', + 'finally', + // End keywords + 'fi', + 'done', + 'end', + 'loop', + 'until', + 'where', + 'when', + ].map(s => s + '\\b') +); +const continuationRegex = new RegExp(`^(${continuations.join('|')})`); + +/** + * Returns true if the given line is a line where we continue completion where + * the indentation level equals the current indentation level. + * + * TODO: Should probably be language specific + */ +function isContinuationLine(line: string) { + return continuationRegex.test(line.trimLeft().toLowerCase()); +} + +/** + * Return the indentation level of a given single line. + * + * If the line is blank, return undefined. + * + * TODO: Possibly support tabs specially? + */ +function indentationOfLine(line: string): number | undefined { + // [^] is used to match any character include '`r', otherwise this regex never matches on + // a file containing Windows newlines. + // TODO this is a bit of hack and ideally we would be using the "right" newline character at the + // point where we split/join lines. + const match = /^(\s*)([^]*)$/.exec(line); + if (match && match[2] && match[2].length > 0) { + return match[1].length; + } else { + return undefined; + } +} + +/** + * Represents the indentation around the context of a cursor position in the code. + * + * The indentation level of the current line is the number of leading whitespace + * characters. If the current line is blank, we define its indentation level to + * be that of the preceding line (recursive if that is also blank). + * + * The indentation level of the next line is defined analogously, but recurses + * forwards until a non-blank line is encountered. It is `undefined` if there + * are no non-blank lines after the current. + */ +export interface ContextIndentation { + /** + * Next smaller indentation above the current line (guaranteed to be + * smaller than `current`, or else undefined). + */ + prev: number | undefined; + /** Indentation at the current line */ + current: number; + /** Indentation at the following line */ + next: number | undefined; +} + +/** + * Return the context indentation corresponding to a given position. + */ +export function contextIndentation(doc: TextDocumentContents, position: IPosition): ContextIndentation { + const source = doc.getText(); + const offset = doc.offsetAt(position); + return contextIndentationFromText(source, offset, doc.detectedLanguageId); +} + +/** + * Return the context indentation corresponding to a given offset in text. + */ +export function contextIndentationFromText(source: string, offset: number, languageId: string): ContextIndentation { + const prevLines = source.slice(0, offset).split('\n'); + const nextLines = source.slice(offset).split('\n'); + function seekNonBlank(lines: string[], start: number, direction: -1 | 1): [number | undefined, number | undefined] { + let i = start; + let ind, + indIdx: number | undefined = undefined; + while (ind === undefined && i >= 0 && i < lines.length) { + ind = indentationOfLine(lines[i]); + indIdx = i; + i += direction; + } + if (languageId === 'python' && direction === -1) { + // HACK: special case to support multi-statement completions after Python doc comments. + // The logic looks for comments formatted as described in PEP 257. + + // The final iteration of the indentation loop will have got us to one before the "current line". + i++; + const trimmedLine = lines[i].trim(); + + if (trimmedLine.endsWith(`"""`)) { + const isSingleLineDocString = trimmedLine.startsWith(`"""`) && trimmedLine !== `"""`; + if (!isSingleLineDocString) { + // Look backwards for the opening """" + i--; + while (i >= 0 && !lines[i].trim().startsWith(`"""`)) { + i--; + } + } + // i should point to the line with the opening """, if found. + // If i is negative then we never found the opening """". Give up and use the indentation + // we originally calculated. + if (i >= 0) { + ind = undefined; + i--; + // This is the same loop as above but specialised for direction = -1 + while (ind === undefined && i >= 0) { + ind = indentationOfLine(lines[i]); + indIdx = i; + i--; + } + } + } + } + return [ind, indIdx]; + } + const [current, currentIdx] = seekNonBlank(prevLines, prevLines.length - 1, -1); + const prev = (() => { + if (current === undefined || currentIdx === undefined) { + return undefined; + } + for (let i = currentIdx - 1; i >= 0; i--) { + const ind = indentationOfLine(prevLines[i]); + if (ind !== undefined && ind < current) { + return ind; + } + } + })(); + const [next] = seekNonBlank(nextLines, 1, 1); // Skip the current line. + return { + prev, + current: current ?? 0, + next, + }; +} + +// If the model thinks we are at the end of a line, do we want to offer a completion +// for the next line? For now (05 Oct 2021) we leave it as false to minimise behaviour +// changes between parsing and indentation mode. +const OfferNextLineCompletion = false; + +/** + * Return an offset where the completion ends its current context, or + * "continue" if it has not yet ended. + * + * A completion should be continued if it is: + * - A very long line that did not yet end; or + * - A multi-line context that is not yet ended. + * + * We use indentation with continuation patterns to determine whether a context + * is ended. + */ +function completionCutOrContinue( + completion: string, + contextIndentation: ContextIndentation, + previewText: string | undefined +): number | 'continue' { + const completionLines = completion.split('\n'); + const isContinuation = previewText !== undefined; + const lastLineOfPreview = previewText?.split('\n').pop(); + let startLine = 0; + if (isContinuation) { + if (lastLineOfPreview?.trim() !== '' && completionLines[0].trim() !== '') { + // If we're in the middle of a line after the preview, we should at least finish it. + startLine++; + } + } + if (!isContinuation && OfferNextLineCompletion && completionLines[0].trim() === '') { + // See the comment on `OfferNextLineCompletion` for why we might do this. + startLine++; + } + if (!isContinuation) { + // We want to offer at least one line. + startLine++; + } + if (completionLines.length === startLine) { + // A single line that did not yet end. + return 'continue'; + } + const breakIndentation = Math.max(contextIndentation.current, contextIndentation.next ?? 0); + for (let i = startLine; i < completionLines.length; i++) { + let line = completionLines[i]; + if (i === 0 && lastLineOfPreview !== undefined) { + line = lastLineOfPreview + line; + } + const ind = indentationOfLine(line); + if (ind !== undefined && (ind < breakIndentation || (ind === breakIndentation && !isContinuationLine(line)))) { + return completionLines.slice(0, i).join('\n').length; + } + } + return 'continue'; +} + +/** + * Returns a callback appropriate as `finishedCb` for + * `CompletionStream.streamChoices` that terminates a block according to + * indentation-logic. + */ +export function indentationBlockFinished( + contextIndentation: ContextIndentation, + previewText: string | undefined +): (completion: string) => number | undefined { + // NOTE: The returned callback is only async because streamChoices needs an + // async callback + return (completion: string) => { + const res = completionCutOrContinue(completion, contextIndentation, previewText); + // streamChoices needs a callback with bad type signature where + // undefined really means "continue". + return res === 'continue' ? undefined : res; + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/prompt.ts b/completions-sample-code/vscode-node/lib/src/prompt/prompt.ts new file mode 100644 index 0000000..bb3e77f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/prompt.ts @@ -0,0 +1,185 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { PromptMetadata } from '../../../prompt/src/components/components'; +import { commentBlockAsSingles } from '../../../prompt/src/languageMarker'; +import { PromptOptions } from '../../../prompt/src/prompt'; +import { SimilarFilesOptions } from '../../../prompt/src/snippetInclusion/similarFiles'; +import { TokenizerName } from '../../../prompt/src/tokenization'; +import { CancellationToken as ICancellationToken } from '../../../types/src'; +import { CompletionState } from '../completionState'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { getNumberOfSnippets, getSimilarFilesOptions } from '../experiments/similarFileOptionsProvider'; +import { getMaxSolutionTokens } from '../openai/openai'; +import { TelemetryWithExp } from '../telemetry'; +import { INotebookCell, INotebookDocument, IntelliSenseInsertion } from '../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../textDocumentManager'; +import { ICompletionsPromptFactoryService } from './completionsPromptFactory/completionsPromptFactory'; +import { ContextProviderTelemetry } from './contextProviderRegistry'; +import { NeighboringFileType, considerNeighborFile } from './similarFiles/neighborFiles'; + +// The minimum number of prompt-eligible characters before we offer a completion +export const MIN_PROMPT_CHARS = 10; + +export interface Prompt { + prefix: string; + suffix: string; + context?: string[]; + prefixTokens?: number; + suffixTokens?: number; + isFimEnabled: boolean; +} + +export interface PromptResponsePresent { + type: 'prompt'; + prompt: Prompt; + /** + * The prefix is sent to the model without trailing whitespace. However the trailing whitespace will + * be kept around to do position adjustments when applying the completion. + */ + trailingWs: string; + computeTimeMs: number; + // evaluate whether we need to keep this. If yes, populate it + neighborSource: Map<NeighboringFileType, string[]>; + metadata: PromptMetadata; + contextProvidersTelemetry?: ContextProviderTelemetry[]; +} + +export interface ExtractPromptOptions { + selectedCompletionInfo?: IntelliSenseInsertion; + data?: unknown; + tokenizer?: TokenizerName; +} + +interface ContextTooShort { + type: 'contextTooShort'; +} +interface CopilotContentExclusion { + type: 'copilotContentExclusion'; +} +interface PromptError { + type: 'promptError'; +} +interface PromptCancelled { + type: 'promptCancelled'; +} + +interface PromptTimeout { + type: 'promptTimeout'; +} + +export const _contextTooShort: ContextTooShort = { type: 'contextTooShort' }; +export const _copilotContentExclusion: CopilotContentExclusion = { type: 'copilotContentExclusion' }; +export const _promptError: PromptError = { type: 'promptError' }; +export const _promptCancelled: PromptCancelled = { type: 'promptCancelled' }; +export const _promptTimeout: PromptTimeout = { type: 'promptTimeout' }; +export type PromptResponse = + | PromptResponsePresent + | CopilotContentExclusion + | ContextTooShort + | PromptError + | PromptCancelled + | PromptTimeout; + +/** Record trailing whitespace, and trim it from prompt if the last line is only whitespace */ +export function trimLastLine(source: string): [string, string] { + const lines = source.split('\n'); + const lastLine = lines[lines.length - 1]; + const extraSpace: number = lastLine.length - lastLine.trimEnd().length; + const promptTrim = source.slice(0, source.length - extraSpace); + const trailingWs = source.slice(promptTrim.length); + const resPrompt = lastLine.length === extraSpace ? promptTrim : source; + return [resPrompt, trailingWs]; +} + +export function extractPrompt( + accessor: ServicesAccessor, + completionId: string, + completionState: CompletionState, + telemetryData: TelemetryWithExp, + cancellationToken?: ICancellationToken, + promptOpts: ExtractPromptOptions = {} +): Promise<PromptResponse> { + const textDocumentManagerService = accessor.get(ICompletionsTextDocumentManagerService); + const notebook = textDocumentManagerService.findNotebook(completionState.textDocument); + const activeCell = notebook?.getCellFor(completionState.textDocument); + if (notebook && activeCell) { + completionState = applyEditsForNotebook(completionState, notebook, activeCell); + } + + telemetryData.extendWithConfigProperties(accessor); + telemetryData.sanitizeKeys(); + const separateContext = true; + const promptFactory = accessor.get(ICompletionsPromptFactoryService); + return promptFactory.prompt( + { + completionId, + completionState, + telemetryData, + promptOpts: { ...promptOpts, separateContext }, + }, + cancellationToken + ); +} + +function addNeighboringCellsToPrompt(neighboringCell: INotebookCell, activeCellLanguageId: string) { + const languageId = neighboringCell.document.detectedLanguageId; + const text = neighboringCell.document.getText(); + if (languageId === activeCellLanguageId) { + // Blocks of the same language are added as is + return text; + } else { + // Consider adding a languageMarker to cells of different languages + // Note, that comments should be added with markers from the language of the active cell! + return commentBlockAsSingles(text, activeCellLanguageId); + } +} + +function applyEditsForNotebook(state: CompletionState, notebook: INotebookDocument, activeCell: INotebookCell) { + const cells = notebook.getCells(); + const beforeCells = cells.filter( + cell => + cell.index < activeCell.index && + considerNeighborFile(activeCell.document.detectedLanguageId, cell.document.detectedLanguageId) + ); + const newText = + beforeCells.length > 0 + ? beforeCells + .map(cell => addNeighboringCellsToPrompt(cell, activeCell.document.detectedLanguageId)) + .join('\n\n') + '\n\n' + : ''; + const top = { line: 0, character: 0 }; + return state.applyEdits([{ newText, range: { start: top, end: top } }]); +} + +export function getPromptOptions(accessor: ServicesAccessor, telemetryData: TelemetryWithExp, languageId: string): PromptOptions { + // Note: the default values of the EXP flags currently overwrite the default `PromptOptions` + const featuresService = accessor.get(ICompletionsFeaturesService); + const maxTokens = featuresService.maxPromptCompletionTokens(telemetryData); + const maxPromptLength = maxTokens - getMaxSolutionTokens(); + + const numberOfSnippets = getNumberOfSnippets(telemetryData, languageId); + const similarFilesOptions: SimilarFilesOptions = getSimilarFilesOptions(accessor, telemetryData, languageId); + + const suffixPercent = featuresService.suffixPercent(telemetryData); + const suffixMatchThreshold = featuresService.suffixMatchThreshold(telemetryData); + + if (suffixPercent < 0 || suffixPercent > 100) { + throw new Error(`suffixPercent must be between 0 and 100, but was ${suffixPercent}`); + } + + if (suffixMatchThreshold < 0 || suffixMatchThreshold > 100) { + throw new Error(`suffixMatchThreshold must be between 0 and 100, but was ${suffixMatchThreshold}`); + } + + return { + maxPromptLength, + similarFilesOptions, + numberOfSnippets, + suffixPercent, + suffixMatchThreshold, + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/emptyRecentEditsProvider.ts b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/emptyRecentEditsProvider.ts new file mode 100644 index 0000000..c80a00a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/emptyRecentEditsProvider.ts @@ -0,0 +1,26 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ICompletionsRecentEditsProviderService } from './recentEditsProvider'; +import { RecentEdit } from './recentEditsReducer'; + +export class EmptyRecentEditsProvider implements ICompletionsRecentEditsProviderService { + declare _serviceBrand: undefined; + isEnabled(): boolean { + return false; + } + + start(): void { + return; + } + + getRecentEdits(): RecentEdit[] { + return []; + } + + getEditSummary(edit: RecentEdit): string | null { + return null; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/recentEditsProvider.ts b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/recentEditsProvider.ts new file mode 100644 index 0000000..aa18ab0 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/recentEditsProvider.ts @@ -0,0 +1,168 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IObservableDocument } from '../../../../../../../platform/inlineEdits/common/observableWorkspace'; +import { autorunWithChanges } from '../../../../../../../platform/inlineEdits/common/utils/observable'; +import { createServiceIdentifier } from '../../../../../../../util/common/services'; +import { Disposable } from '../../../../../../../util/vs/base/common/lifecycle'; +import { mapObservableArrayCached } from '../../../../../../../util/vs/base/common/observableInternal'; +import { ICompletionsObservableWorkspace } from '../../completionsObservableWorkspace'; +import { + getAllRecentEditsByTimestamp, + RecentEdit, + RecentEditMap, + recentEditsReducer, + summarizeEdit, +} from './recentEditsReducer'; + +export const ICompletionsRecentEditsProviderService = createServiceIdentifier<ICompletionsRecentEditsProviderService>('ICompletionsRecentEditsProviderService'); +export interface ICompletionsRecentEditsProviderService { + readonly _serviceBrand: undefined; + isEnabled(): boolean; + getRecentEdits(): RecentEdit[]; + getEditSummary(edit: RecentEdit): string | null; + start(): void; +} + +export interface RecentEditsConfig { + // the maximum number of recent files to include in the prompt + maxFiles: number; + // the number of recent edits to include in the prompt + maxEdits: number; + // the number of context lines around the edit to include in the prompt + diffContextLines: number; + // the distance between edits to merge them into one edit + editMergeLineDistance: number; + // the maximum number of characters per edit to include in the prompt + maxCharsPerEdit: number; + // the debounce timeout for tracking recent edits + debounceTimeout: number; + // the type of summarization we use for recent edits + summarizationFormat: string; + // whether to remove deleted lines from diff in the prompt + removeDeletedLines: boolean; + // whether to organize insertions before deletions in the diff format + insertionsBeforeDeletions: boolean; + // whether to append a no-reply marker to the end of deletions in the diff format + appendNoReplyMarker: boolean; + // the filtered-out window limit between active-file recent edits and cursor + activeDocDistanceLimitFromCursor: number | undefined; + // the maximum number of lines per edit to include in the prompt + maxLinesPerEdit: number; +} + +const RECENT_EDITS_DEFAULT_CONFIG: RecentEditsConfig = Object.freeze({ + maxFiles: 20, + maxEdits: 8, + diffContextLines: 3, + editMergeLineDistance: 1, + maxCharsPerEdit: 2000, + debounceTimeout: 500, + summarizationFormat: 'diff', + removeDeletedLines: false, + insertionsBeforeDeletions: true, + appendNoReplyMarker: true, + activeDocDistanceLimitFromCursor: 100, + maxLinesPerEdit: 10, +}); + +export class FullRecentEditsProvider extends Disposable implements ICompletionsRecentEditsProviderService { + declare _serviceBrand: undefined; + + private _started: boolean = false; + private recentEditMap: RecentEditMap = {}; + private recentEdits: RecentEdit[] = []; + private recentEditSummaries: WeakMap<RecentEdit, string | null> = new WeakMap(); + private debounceTimeouts: { [key: string]: TimeoutHandle } = {}; + private readonly _config: RecentEditsConfig; + + constructor( + config: RecentEditsConfig | undefined, + @ICompletionsObservableWorkspace private readonly observableWorkspace: ICompletionsObservableWorkspace, + ) { + super(); + this._config = config ?? Object.assign({}, RECENT_EDITS_DEFAULT_CONFIG); + } + + get config(): RecentEditsConfig { + return this._config; + } + + isEnabled(): boolean { + return true; + } + + getRecentEdits(): RecentEdit[] { + return this.recentEdits; + } + + getEditSummary(edit: RecentEdit): string | null { + return this.recentEditSummaries.get(edit) ?? null; + } + + protected updateRecentEdits(docId: string, newContents: string): void { + this.recentEditMap = recentEditsReducer(this.recentEditMap, docId, newContents, this._config); + this.recentEdits = getAllRecentEditsByTimestamp(this.recentEditMap); + + this.recentEdits.forEach(edit => { + if (!this.recentEditSummaries.has(edit)) { + // Generate a summary for the edit if it doesn't already exist + const summary = summarizeEdit(edit, this._config); + this.recentEditSummaries.set(edit, summary); + } + }); + } + + start() { + // By the default, the provider starts lazily on the first completion request. + if (this._started) { + return; + } + this._started = true; + + mapObservableArrayCached( + this, + this.observableWorkspace.openDocuments, + (doc: IObservableDocument, store) => { + store.add( + autorunWithChanges( + this, + { + value: doc.value, + selection: doc.selection, + languageId: doc.languageId, + }, + data => { + if (data.value.changes.length > 0) { + const prevText = data.value.previous?.value; + const newText = data.value.value.value; + const docId = doc.id.toString(); + + // clear any existing debounce timeout for this document + // note that you can call clearTimeout on undefined, so we don't need to check if it exists + clearTimeout(this.debounceTimeouts[docId]); + + if (!this.recentEditMap[docId] && prevText) { + // This is the first time the edit is being stored, but we also know what the previous text was. + // We need to add the previous text to the reducer so that we can get a diff. + this.updateRecentEdits(docId, prevText); + } else if (this._config.debounceTimeout === 0) { + // allow setting debounce to 0 in experiments / settings for immediate updates + this.updateRecentEdits(docId, newText); + } else { + // update in a few milliseconds + this.debounceTimeouts[docId] = setTimeout(() => { + this.updateRecentEdits(docId, newText); + }, this._config.debounceTimeout ?? 500); + } + } + } + ) + ); + }, + d => d.id + ).recomputeInitiallyAndOnChange(this._store); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/recentEditsReducer.ts b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/recentEditsReducer.ts new file mode 100644 index 0000000..116cc6e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/recentEditsReducer.ts @@ -0,0 +1,451 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { RecentEditsConfig } from './recentEditsProvider'; + +/** The shape of one unified-diff hunk, independent of formatting. */ +export interface DiffHunk { + file: string; + // the first line index (0-based) of the context window + pre: number; + // one past the last line index of the context window + post: number; + // context before the change + before: string[]; + // lines removed + removed: string[]; + // lines added + added: string[]; + // context after the change + after: string[]; +} + +export interface RecentEdit { + file: string; + startLine: number; + endLine: number; + diff: DiffHunk; + timestamp: number; +} + +export type RecentEditMap = Record<string, { originalContent: string; currentContent: string; edits: RecentEdit[] }>; + +/** + * Flatten all edits from a RecentEditMap into a single array, + * sorted by timestamp (oldest first). + */ +export function getAllRecentEditsByTimestamp(map: RecentEditMap): RecentEdit[] { + return Object.values(map) + .flatMap(fileEntry => fileEntry.edits) + .sort((a, b) => a.timestamp - b.timestamp); +} + +/** + * Find the first/last differing line indices. + * Returns null if the two are identical. + */ +export function findChangeSpan( + prevLines: string[], + newLines: string[] +): { start: number; endPrev: number; endNew: number } | null { + let start = 0; + while (start < prevLines.length && start < newLines.length && prevLines[start] === newLines[start]) { + start++; + } + + let endPrev = prevLines.length - 1; + let endNew = newLines.length - 1; + while (endPrev >= start && endNew >= start && prevLines[endPrev] === newLines[endNew]) { + endPrev--; + endNew--; + } + + // truly identical + if (start > endPrev && start > endNew) { return null; } + + return { start, endPrev, endNew }; +} + +/** + * Collect everything needed to render a single diff in any format. + */ +export function getDiff( + file: string, + prevLines: string[], + newLines: string[], + start: number, + endPrev: number, + endNew: number, + context: number +): DiffHunk { + const pre = Math.max(0, start - context); + const post = Math.min(newLines.length, endNew + context + 1); + + return { + file, + pre, + post, + before: prevLines.slice(pre, start), + removed: prevLines.slice(start, endPrev + 1), + added: newLines.slice(start, endNew + 1), + after: newLines.slice(endNew + 1, post), + }; +} + +/** + * Calculates the number of characters in a DiffHunk. This includes context, removed, and added lines, but no formatting. + * @param hunk A DiffHunk object containing the diff information. + * @returns The total number of characters in the diff. + */ +function measureDiffSize(hunk: DiffHunk): number { + // Calculate the size of the diff by summing the lengths of all lines + // in the before, removed, added, and after sections. + const allLines = [...hunk.before, ...hunk.removed, ...hunk.added, ...hunk.after]; + return allLines.reduce((acc, line) => acc + line.length + 1, 0); +} + +/** + * Turn a DiffHunk into a standard unified diff string. + */ +export function unifiedDiff( + hunk: DiffHunk, + removeDeletedLines: boolean = false, + insertionsBeforeDeletions: boolean = false, + appendNoReplyMarker: boolean = false +): string { + const out: string[] = []; + + out.push(`--- a/${hunk.file}`); + out.push(`+++ b/${hunk.file}`); + const oldLen = hunk.before.length + hunk.removed.length + hunk.after.length; + const newLen = hunk.before.length + hunk.added.length + hunk.after.length; + out.push(`@@ -${hunk.pre + 1},${oldLen} +${hunk.pre + 1},${newLen} @@`); + + for (const line of hunk.before) { out.push(' ' + line); } + if (insertionsBeforeDeletions) { + for (const line of hunk.added) { out.push('+' + line); } + } + if (!removeDeletedLines) { + const deletedLinesSpecialText = appendNoReplyMarker ? ' --- IGNORE ---' : ''; + for (const line of hunk.removed) { out.push('-' + line + deletedLinesSpecialText); } + } + if (!insertionsBeforeDeletions) { + for (const line of hunk.added) { out.push('+' + line); } + } + for (const line of hunk.after) { out.push(' ' + line); } + + return out.join('\n') + '\n'; +} + +/** + * Turn a DiffHunk into an Aider's Diff string. OpenAI recommends this for 4.1 models. https://aider.chat/docs/more/edit-formats.html#diff + */ +function aidersDiff(hunk: DiffHunk, removeDeletedLines = false): string { + const { before, removed, added, after } = hunk; + const res: string[] = []; + + res.push('>>>>>>> SEARCH'); + res.push(...before); + if (removeDeletedLines) { + res.push('...'); + } else { + res.push(...removed); + } + res.push(...after); + + res.push('======='); + + res.push(...before); + res.push(...added); + res.push(...after); + + res.push('<<<<<<<<< REPLACE'); + return res.join('\n'); +} + +/** + * Turn a DiffHunk into a plain english find/replace string + */ +export function findReplaceDiff(hunk: DiffHunk, removeDeletedLines = false): string { + const { before, removed, added, after } = hunk; + const removedWithWarning = removeDeletedLines + ? ['...'] + : removed.map(line => `${line} --- DO NOT REPLY WITH CODE FROM THIS LINE ---`); + + const beforeSection = [...before, ...removedWithWarning, ...after]; + const afterSection = [...before, ...added, ...after]; + + const res: string[] = []; + res.push('--- User edited code: ---'); + res.push(...beforeSection); + + if (removedWithWarning.length === 0) { + res.push(`--- and added ${added.length} line${added.length === 1 ? '' : 's'} to make: ---`); + } else if (added.length === 0) { + res.push( + `--- and deleted ${removedWithWarning.length} line${removedWithWarning.length === 1 ? '' : 's'} to make: ---` + ); + } else { + res.push('--- and replaced it with: ---'); + } + + res.push(...afterSection); + res.push('--- End of edit ---'); + return res.join('\n'); +} + +/** Apply a sequence of edits to a lines-array, in order. */ +function applyEditsToLines(lines: string[], edits: RecentEdit[]): string[] { + for (const e of edits) { + const before = lines.slice(0, e.startLine); + const after = lines.slice(e.endLine + 1); + const insert = e.diff.added ? e.diff.added : []; + lines = [...before, ...insert, ...after]; + } + return lines; +} + +/** + * Determines whether two edits overlap or are close enough to be considered adjacent. + * + * @param incoming - The new edit being evaluated, represented as a `RecentEdit` object. + * @param last - The most recent edit already processed, represented as a `RecentEdit` object. + * @param editMergeLineDistance - The maximum number of lines between edits to consider them adjacent. + * @returns `true` if the edits overlap or are within the specified line distance; otherwise, `false`. + */ +export function editsOverlap(incoming: RecentEdit, last: RecentEdit, editMergeLineDistance: number): boolean { + const { added } = last.diff; + const lastStart = last.startLine; + const lastEnd = last.startLine + added.length; + const incStart = incoming.startLine; + const incEnd = incoming.endLine + 1; + + // Two ranges overlap (or are within the merge distance) if + // the start of one is no more than `editMergeLineDistance` after the end of the other, and vice versa. + return incStart <= lastEnd + editMergeLineDistance && incEnd >= lastStart - editMergeLineDistance; +} + +/** + * Add an incoming hunk, coalesce overlaps, and immediately trim+rebase if needed. + */ +export function updateEdits( + originalContent: string, + existing: RecentEdit[], + incoming: RecentEdit, + currentFileLines: string[], + config: RecentEditsConfig +): { originalContent: string; edits: RecentEdit[] } { + let edits = [...existing]; + + // Try to merge only if the ranges actually overlap + if (edits.length > 0) { + const last = edits[edits.length - 1]; + const overlaps = editsOverlap(incoming, last, config.editMergeLineDistance); + + if (overlaps) { + // build the file state _just before_ the incoming edit + const prevLines = applyEditsToLines(originalContent.split('\n'), edits.slice(0, -1)); + + // compute the true minimal span + const span = findChangeSpan(prevLines, currentFileLines); + if (span) { + // re-build a single merged hunk + incoming = buildIncomingEdit(incoming.file, prevLines, currentFileLines, span, config); + edits = [...edits.slice(0, -1), incoming]; + } + // else a no-op or perfect revert, just drop the incoming + } else { + edits.push(incoming); + } + } else { + edits.push(incoming); + } + + // Trim & rebase _after_ appending/merging, so incoming is folded + if (edits.length > config.maxEdits) { + // Push out the stale edits + const staleEdits = edits.slice(0, edits.length - config.maxEdits); + edits = edits.slice(edits.length - config.maxEdits, edits.length); + const allLines = applyEditsToLines(originalContent.split('\n'), staleEdits); + originalContent = allLines.join('\n'); + } + + return { originalContent, edits }; +} + +/** Build the incoming edit object */ +export function buildIncomingEdit( + file: string, + prevLines: string[], + nextLines: string[], + span: { start: number; endPrev: number; endNew: number }, + config: RecentEditsConfig +): RecentEdit { + const { start, endPrev, endNew } = span; + if (!config || typeof config.diffContextLines !== 'number') { + throw new Error('Invalid configuration passed to buildIncomingEdit'); + } + const diff = getDiff(file, prevLines, nextLines, start, endPrev, endNew, config.diffContextLines); + + return { + file, + startLine: start, + endLine: endPrev, + diff, + timestamp: performance.now(), + }; +} + +/** + * Trim old files from the state. + */ +export function trimOldFilesFromState(state: RecentEditMap, maxFiles: number): RecentEditMap { + const newState = { ...state }; + + const modifiedFilesInOrder = Object.entries(state) + // take only modified files + .filter(([fileName]) => state[fileName].edits.length) + // sort by timestamp of most recent edit + .sort( + ([aFile, a], [bFile, b]) => a.edits[a.edits.length - 1].timestamp - b.edits[b.edits.length - 1].timestamp + ); + + const filesToTrim = Math.max(0, modifiedFilesInOrder.length - maxFiles); + if (filesToTrim) { + for (let i = 0; i < filesToTrim; i++) { + const fileName = modifiedFilesInOrder[i][0]; + delete newState[fileName]; + } + } + + return newState; +} + +/** + * Reducer that takes a file and its new contents, + * merging them into a clean structure of recent edits. + */ +export function recentEditsReducer( + state: RecentEditMap = {}, + file: string, + newContents: string, + config: RecentEditsConfig +): RecentEditMap { + if (newContents.length > 2 * 1024 * 1024) { + // don't try to track files larger than 2mb (around 100k lines) + return state; + } + + const prev = state[file]; + + // first time we see this file + if (!prev) { + return { + ...state, + [file]: { + originalContent: newContents, + currentContent: newContents, + edits: [], + }, + }; + } + + // nothing changed + if (prev.currentContent === newContents) { + return state; + } + + const prevLines = prev.currentContent.split('\n'); + const newLines = newContents.split('\n'); + + // detect the changed span + const span = findChangeSpan(prevLines, newLines); + if (!span) { + // content drifted back to identical + return { + ...state, + [file]: { ...prev, currentContent: newContents }, + }; + } + + // build the single incoming edit + const incoming = buildIncomingEdit(file, prevLines, newLines, span, config); + if (measureDiffSize(incoming.diff) > config.maxCharsPerEdit) { + // User is making a huge edit, so we just reset the state for this file. + // This is a performance optimization to avoid keeping large diffs in memory. + return { + ...state, + [file]: { + originalContent: newContents, + currentContent: newContents, + edits: [], + }, + }; + } + + // merge/trim/rebase all at once + const { originalContent: updatedOriginal, edits: updatedEdits } = updateEdits( + prev.originalContent, + prev.edits, + incoming, + newLines, + config + ); + + // update the state for this file + const stateWithLatestEdit = { + ...state, + [file]: { + originalContent: updatedOriginal, + currentContent: newContents, + edits: updatedEdits, + }, + }; + + // Trim old files if needed. Need to do this _after_ the new edit was added, since the + // timestamp of this file will have changed. + return trimOldFilesFromState(stateWithLatestEdit, config.maxFiles); +} + +/** + * Summarizes a single recent edit for the prompt. + * @param edit + * @param config RecentEditsPromptConfig + * @returns a string summarizing the edit for the prompt, or null if the edit should be left out + */ +export function summarizeEdit(edit: RecentEdit, config: RecentEditsConfig): string | null { + const oldNonEmptyLines: string[] = edit.diff.removed.filter(x => x.trim().length > 0); + const newNonEmptyLines: string[] = edit.diff.added.filter(x => x.trim().length > 0); + + let result: string | null; + if (config.removeDeletedLines && newNonEmptyLines.length === 0) { + // skip over a diff that has only deleted lines + result = null; + } else if (oldNonEmptyLines.length === 0 && newNonEmptyLines.length === 0) { + // skip over a diff which would only contain -/+ without any content + result = null; + } else if (oldNonEmptyLines.join('').trim() === newNonEmptyLines.join('').trim()) { + // skip over a diff that has only whitespace changes + result = null; + } else if (edit.diff.added.length > config.maxLinesPerEdit || edit.diff.removed.length > config.maxLinesPerEdit) { + // skip over a diff that is too large line-wise + result = null; + } else if (config.summarizationFormat === 'aiders-diff') { + result = aidersDiff(edit.diff); + } else if (config.summarizationFormat === 'diff') { + result = unifiedDiff( + edit.diff, + config.removeDeletedLines, + config.insertionsBeforeDeletions, + config.appendNoReplyMarker + ); + } else if (config.summarizationFormat === 'find-replace') { + result = findReplaceDiff(edit.diff); + } else { + throw new Error(`Unknown summarization format: ${config.summarizationFormat}`); + } + + return result; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/test/recentEditsReducer.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/test/recentEditsReducer.test.ts new file mode 100644 index 0000000..5b67e0f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/recentEdits/test/recentEditsReducer.test.ts @@ -0,0 +1,1001 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { readFileSync } from 'fs'; +import { ComplexityData, determineTimeComplexity } from '../../test/determineTimeComplexity'; +import { RecentEditsConfig } from '../recentEditsProvider'; +import { + buildIncomingEdit, + DiffHunk, + editsOverlap, + findChangeSpan, + findReplaceDiff, + getAllRecentEditsByTimestamp, + getDiff, + RecentEdit, + RecentEditMap, + recentEditsReducer, + summarizeEdit, + trimOldFilesFromState, + unifiedDiff, + updateEdits, +} from '../recentEditsReducer'; + +// Note, that this configuration is only used in testing, and is different from the one used in production. +const config: RecentEditsConfig = { + maxFiles: 5, + maxEdits: 5, + diffContextLines: 3, + editMergeLineDistance: 3, + maxCharsPerEdit: 2000, + debounceTimeout: 500, + + summarizationFormat: 'diff', + removeDeletedLines: false, + insertionsBeforeDeletions: false, + appendNoReplyMarker: false, + activeDocDistanceLimitFromCursor: 100, + maxLinesPerEdit: 10, +}; + +suite('findChangeSpan', function () { + test('no differences returns null', function () { + const originalLines = ['a', 'b', 'c']; + assert.strictEqual(findChangeSpan(originalLines, [...originalLines]), null); + }); + + test('simple replacement span', function () { + const prevLines = ['a', 'b', 'c']; + const nextLines = ['a', 'B', 'c']; + const span = findChangeSpan(prevLines, nextLines)!; + assert.deepStrictEqual(span, { start: 1, endPrev: 1, endNew: 1 }); + }); + + test('insertion at top', function () { + const prevLines = ['a', 'b', 'c']; + const nextLines = ['X', 'a', 'b', 'c']; + const span = findChangeSpan(prevLines, nextLines)!; + assert.deepStrictEqual(span, { start: 0, endPrev: -1, endNew: 0 }); + }); +}); + +suite('helper: editsOverlap', function () { + test('overlap when end of incoming is within last start', function () { + const incoming0 = { + startLine: 1, + endLine: 2, + diff: { added: ['x', 'y'], removed: ['a', 'b'] }, + } as unknown as RecentEdit; + const last0 = { + startLine: 2, + endLine: 3, + diff: { added: [], removed: [] }, + } as unknown as RecentEdit; + + // Should overlap because lines are adjacent and merge distance is 0 + assert.strictEqual(editsOverlap(incoming0, last0, 0), true); + }); + + test('overlap when start of incoming is within last edit', function () { + const incoming1 = { + startLine: 1, + endLine: 1, + diff: { added: ['x', 'y'], removed: [] }, + } as unknown as RecentEdit; + const last1 = { + startLine: 2, + endLine: 2, + diff: { added: ['aaaaa'], removed: [] }, + } as unknown as RecentEdit; + + // Should overlap because lines are adjacent and merge distance is 0 + assert.strictEqual(editsOverlap(incoming1, last1, 0), true); + }); + + test('overlap when incoming is entirely within last edit', function () { + const incoming2 = { + startLine: 1, + endLine: 2, + diff: { added: ['x', 'y'], removed: [] }, + } as unknown as RecentEdit; + const last2 = { + startLine: 0, + endLine: 3, + diff: { added: ['a', 'b', 'c'], removed: ['1', '2', '3'] }, + } as unknown as RecentEdit; + + // Should overlap because incoming edit is entirely within last edit + assert.strictEqual(editsOverlap(incoming2, last2, 0), true); + }); +}); + +suite('updateEdits (merge & trim)', function () { + const baseLines = ['a', 'b', 'c', 'd']; + const baseContent = baseLines.join('\n'); + const v2Lines = ['a', 'B', 'c', 'd']; + + // initial incoming edit: replace b->B + const firstSpan = { start: 1, endPrev: 1, endNew: 1 } as const; + const incoming1 = buildIncomingEdit('f', baseLines, v2Lines, firstSpan, config); + + test('merges into empty list', function () { + const { originalContent, edits } = updateEdits(baseContent, [] as RecentEdit[], incoming1, v2Lines, config); + + assert.strictEqual(edits.length, 1); + assert.strictEqual(originalContent, baseContent); + + const diffText = unifiedDiff(edits[0].diff); + const expected = + ` +--- a/f ++++ b/f +@@ -1,4 +1,4 @@ + a +-b ++B + c + d +`.trim() + '\n'; + + assert.strictEqual(diffText, expected); + }); + + test('coalesces overlap', function () { + const { edits: existing } = updateEdits(baseContent, [] as RecentEdit[], incoming1, v2Lines, config); + + const v3Lines = ['a', 'B', 'OH NO', 'c', 'd']; + const span2 = findChangeSpan(v2Lines, v3Lines)!; + const incoming2 = buildIncomingEdit('f', v2Lines, v3Lines, span2, config); + + const { originalContent: oc2, edits } = updateEdits(baseContent, existing, incoming2, v3Lines, config); + assert.strictEqual(edits.length, 1, 'should merge two overlapping edits'); + assert.strictEqual(oc2, baseContent); + + const diffText = unifiedDiff(edits[0].diff); + assert.ok(diffText.includes('-b')); + assert.ok(diffText.includes('+B')); + assert.ok(diffText.includes('+OH NO')); + }); + + test('trims and rebases when exceeding MAX_EDITS', function () { + const initialLines = Array.from({ length: 100 }, (_, i) => `line${i}`); + let original = initialLines.join('\n'); + let edits: RecentEdit[] = []; + + for (let i = 0; i < 70; i += 10) { + const prev = original.split('\n'); + const modifiedLine = `new${i}`; + const next = [...prev]; + next[i] = modifiedLine; + const span = findChangeSpan(prev, next)!; + const incoming = buildIncomingEdit('f', prev, next, span, config); + const result = updateEdits(original, edits, incoming, next, config); + original = result.originalContent; + edits = result.edits; + } + + assert.strictEqual(edits.length, 5, 'should cap edits to MAX_EDITS'); + assert.ok(original.split('\n').includes('new10'), 'original text should include line from 6 edits prior'); + assert.ok(!original.split('\n').includes('new20'), 'snapshot text should not include line from 5 edits prior'); + }); +}); + +suite('updateEdits nearby hunk merging', function () { + const cases = [ + { sep: 0, expected: 1 }, + { sep: 1, expected: 1 }, + { sep: 2, expected: 1 }, + { sep: 3, expected: 1 }, + { sep: 4, expected: 2 }, + { sep: 5, expected: 2 }, + ]; + + cases.forEach(({ sep, expected }) => { + for (const position of ['above', 'below']) { + test(`edit ${sep} line${sep === 1 ? '' : 's'} ${position} previous edit ${expected === 1 ? 'merges into one hunk' : 'becomes a separate hunk'}`, function () { + const base = Array.from({ length: 10 }, (_, i) => i.toString()); + let orig = base.join('\n'); + let edits: RecentEdit[] = []; + + // get positions of where each edit will begin + const editFirstLines = [2, 3 + sep]; + if (position === 'above') { + // reverse the order of edits when making edits bottom to top + editFirstLines.reverse(); + } + + // make first edit + const prev1 = base; + const next1 = [...prev1]; + next1[editFirstLines[0]] = 'X'; + const span1 = findChangeSpan(prev1, next1)!; + const h1 = buildIncomingEdit('f', prev1, next1, span1, config); + ({ originalContent: orig, edits } = updateEdits(orig, edits, h1, next1, config)); + + // second edit separated by sep lines + const prev2 = next1; + const next2 = [...prev2]; + next2[editFirstLines[1]] = 'Y'; + const span2 = findChangeSpan(prev2, next2)!; + const h2 = buildIncomingEdit('f', prev2, next2, span2, config); + const res = updateEdits(orig, edits, h2, next2, config); + + assert.strictEqual( + res.edits.length, + expected, + `separated by ${sep} lines should ${expected === 1 ? 'merge' : 'split'}` + ); + }); + } + }); +}); + +suite('updateEdits overlapping multi-line edits', function () { + test('partially overlapping multi-line edits merge into single hunk', function () { + const base = ['1', '2', '3', '4', '5']; + const orig = base.join('\n'); + // first edit: change lines 1-2 + const v1 = ['1X', '2X', '3', '4', '5']; + const span1 = findChangeSpan(base, v1)!; + let { originalContent, edits } = updateEdits( + orig, + [], + buildIncomingEdit('f', base, v1, span1, config), + v1, + config + ); + // second edit: change lines 2-3 (overlaps at index 1) + const v2 = ['1X', '2X', '3', '4Y', '5Y']; + const span2 = findChangeSpan(v1, v2)!; + ({ originalContent, edits } = updateEdits( + originalContent, + edits, + buildIncomingEdit('f', v1, v2, span2, config), + v2, + config + )); + assert.strictEqual(edits.length, 1); + const diffLines = unifiedDiff(edits[0].diff).split('\n'); + assert.deepEqual(diffLines, [ + '--- a/f', + '+++ b/f', + '@@ -1,5 +1,5 @@', + '-1', + '-2', + '-3', + '-4', + '-5', + '+1X', + '+2X', + '+3', + '+4Y', + '+5Y', + '', + ]); + }); + + test('multi-line edit containing a smaller multi-line edit merges into original span', function () { + const base = ['A', 'B', 'C', 'D', 'E']; + const orig = base.join('\n'); + // large edit: B,C,D -> x,y,z + const v1 = ['A', 'x', 'y', 'z', 'E']; + const span1 = findChangeSpan(base, v1)!; + let { originalContent, edits } = updateEdits( + orig, + [], + buildIncomingEdit('f', base, v1, span1, config), + v1, + config + ); + // smaller edit inside that: y -> Y + const v2 = ['A', 'x', 'Y', 'z', 'E']; + const span2 = findChangeSpan(v1, v2)!; + ({ originalContent, edits } = updateEdits( + originalContent, + edits, + buildIncomingEdit('f', v1, v2, span2, config), + v2, + config + )); + assert.strictEqual(edits.length, 1); + const { diff } = edits[0]; + // removed should be original B,C,D + assert.deepStrictEqual(diff.removed, ['B', 'C', 'D']); + // added should reflect x, Y, z + assert.deepStrictEqual(diff.added, ['x', 'Y', 'z']); + }); + + test('multi-line delete followed by multi-line insert', function () { + const base = Array.from({ length: 50 }, (_, i) => i.toString()); + const orig = base.join('\n'); + + // large deletion + const v1 = [...base.slice(0, 20), ...base.slice(30)]; + const span1 = findChangeSpan(base, v1)!; + let { originalContent, edits } = updateEdits( + orig, + [], + buildIncomingEdit('f', base, v1, span1, config), + v1, + config + ); + + // insert + const v2 = [...base.slice(0, 20), 'new line', 'another new line', ...base.slice(30)]; + const span2 = findChangeSpan(v1, v2)!; + ({ originalContent, edits } = updateEdits( + originalContent, + edits, + buildIncomingEdit('f', v1, v2, span2, config), + v2, + config + )); + + // should become one replace + assert.strictEqual(edits.length, 1); + const { diff } = edits[0]; + assert.deepEqual(diff.added, ['new line', 'another new line']); + assert.equal(diff.removed.length, 10); + }); +}); + +suite('unifiedDiff', function () { + test('formats a simple replacement with 1 line of context by default', function () { + const before = ['line1', 'line2', 'line3']; + const after = ['line1', 'line2 modified', 'line3']; + + const span = findChangeSpan(before, after); + assert.notStrictEqual(span, null); + const { start, endPrev, endNew } = span!; + + const hunk = getDiff('f', before, after, start, endPrev, endNew, 1); + const lines = unifiedDiff(hunk).trim().split('\n'); + + assert.deepStrictEqual(lines, [ + '--- a/f', + '+++ b/f', + '@@ -1,3 +1,3 @@', + ' line1', + '-line2', + '+line2 modified', + ' line3', + ]); + }); + + test('still shows removed lines even if you once wanted to "remove" them', function () { + const before = ['line1', 'line2', 'line3']; + const after = ['line1', 'line2 modified', 'line3']; + + const span = findChangeSpan(before, after)!; + const hunk = getDiff('f', before, after, span.start, span.endPrev, span.endNew, 1); + const lines = unifiedDiff(hunk).trim().split('\n'); + + assert.deepStrictEqual(lines, [ + '--- a/f', + '+++ b/f', + '@@ -1,3 +1,3 @@', + ' line1', + '-line2', + '+line2 modified', + ' line3', + ]); + }); + + test('returns null from findChangeSpan when there are truly no changes', function () { + const before = ['line1', 'line2', 'line3']; + const after = ['line1', 'line2', 'line3']; + assert.strictEqual(findChangeSpan(before, after), null); + }); + + test('detects even pure whitespace changes', function () { + const before = ['line1', 'line2 ', 'line3']; + const after = ['line1', 'line2', 'line3']; + + const span = findChangeSpan(before, after); + assert.notStrictEqual(span, null); + const { start, endPrev, endNew } = span!; + + const hunk = getDiff('file.txt', before, after, start, endPrev, endNew, 1); + const lines = unifiedDiff(hunk).trim().split('\n'); + + assert.deepStrictEqual(lines, [ + '--- a/file.txt', + '+++ b/file.txt', + '@@ -1,3 +1,3 @@', + ' line1', + '-line2 ', + '+line2', + ' line3', + ]); + }); +}); + +suite('findReplaceDiff', function () { + test('wraps removed lines in the "DO NOT REPLY" marker and shows the replacement', function () { + const before = ['line1', 'line2', 'line3']; + const after = ['line1', 'line2 modified', 'line3']; + + const span = findChangeSpan(before, after)!; + const hunk = getDiff('f', before, after, span.start, span.endPrev, span.endNew, 1); + const lines = findReplaceDiff(hunk).split('\n'); + + assert.deepStrictEqual(lines, [ + '--- User edited code: ---', + 'line1', + 'line2 --- DO NOT REPLY WITH CODE FROM THIS LINE ---', + 'line3', + '--- and replaced it with: ---', + 'line1', + 'line2 modified', + 'line3', + '--- End of edit ---', + ]); + }); + + test('insertion-only case shows "added" message', function () { + const before = ['a', 'b']; + const after = ['a', 'b', 'c', 'd']; + const span = { start: 2, endPrev: 1, endNew: 3 } as const; + const hunk = getDiff('f', before, after, span.start, span.endPrev, span.endNew, 1); + const lines = findReplaceDiff(hunk).split('\n'); + assert.ok(lines.includes('--- and added 2 lines to make: ---')); + }); + + test('deletion-only case shows "deleted" message', function () { + const before = ['a', 'b', 'c']; + const after = ['a', 'c']; + const span = { start: 1, endPrev: 1, endNew: 0 } as const; + const hunk = getDiff('f', before, after, span.start, span.endPrev, span.endNew, 1); + const lines = findReplaceDiff(hunk).split('\n'); + assert.ok(lines.includes('--- and deleted 1 line to make: ---')); + }); +}); + +suite('recentEditsReducer', function () { + test('merges replacement + insert into one hunk when adjacent', function () { + const file = 'ex.txt'; + const v1 = ['a', 'b', 'c'].join('\n'); + const v2 = ['a', 'B', 'c'].join('\n'); + const v3 = ['X', 'a', 'B', 'c'].join('\n'); + let state: RecentEditMap = {}; + state = recentEditsReducer(state, file, v1, config); + state = recentEditsReducer(state, file, v2, config); + state = recentEditsReducer(state, file, v3, config); + + assert.strictEqual(state[file].edits.length, 1); + + const diffText = unifiedDiff(state[file].edits[0].diff); + const expected = + ` +--- a/ex.txt ++++ b/ex.txt +@@ -1,3 +1,4 @@ +-a +-b ++X ++a ++B + c +`.trim() + '\n'; + assert.strictEqual(diffText, expected); + }); + + test('does not merge distant edits into one hunk (separated by 5 lines)', function () { + const file = 'far.txt'; + // 8-line file so edits at index 1 and 7 are separated by 5 unmodified lines + const base = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']; + const v1 = base.join('\n'); + const v2 = [...base]; + v2[1] = 'B'; + const v3 = [...v2]; + v3[7] = 'H'; + + let state: RecentEditMap = {}; + state = recentEditsReducer(state, file, v1, config); + state = recentEditsReducer(state, file, v2.join('\n'), config); + state = recentEditsReducer(state, file, v3.join('\n'), config); + + // edits are far enough apart (5 lines), still two hunks + assert.strictEqual(state[file].edits.length, 2); + + const diffs = state[file].edits.map(e => unifiedDiff(e.diff)); + + const expectedDiff1 = + ` +--- a/far.txt ++++ b/far.txt +@@ -1,5 +1,5 @@ + a +-b ++B + c + d + e +`.trim() + '\n'; + + const expectedDiff2 = + ` +--- a/far.txt ++++ b/far.txt +@@ -5,4 +5,4 @@ + e + f + g +-h ++H +`.trim() + '\n'; + + assert.strictEqual(diffs[0], expectedDiff1); + assert.strictEqual(diffs[1], expectedDiff2); + }); + + test('two consecutive edits in same spot do merge', function () { + const file = 'test.txt'; + const v0 = ['A', 'B', 'C', 'D', 'E'].join('\n'); + let state: RecentEditMap = {}; + state = recentEditsReducer(state, file, v0, config); + assert.strictEqual(state[file].edits.length, 0); + + const v1 = ['A', 'B', 'C', 'D', 'e'].join('\n'); + state = recentEditsReducer(state, file, v1, config); + assert.strictEqual(state[file].edits.length, 1); + + const v2 = ['A', 'B', 'C', 'D', 'E'].join('\n'); + state = recentEditsReducer(state, file, v2, config); + assert.strictEqual(state[file].edits.length, 1); + }); + + test('maintains a maximum number of files in the state', function () { + let state: RecentEditMap = {}; + for (let i = 0; i < 10; i++) { + const file = `file${i}.txt`; + state = recentEditsReducer(state, file, 'a\nb\nc\n', config); + state = recentEditsReducer(state, file, 'a\nb\nc\nd\n', config); + } + + assert.deepEqual(Object.keys(state).sort(), ['file5.txt', 'file6.txt', 'file7.txt', 'file8.txt', 'file9.txt']); + }); + + test('keeps separate edit lists for each file', function () { + let state: RecentEditMap = {}; + state = recentEditsReducer(state, 'a.txt', 'A', config); + state = recentEditsReducer(state, 'b.txt', 'B', config); + state = recentEditsReducer(state, 'a.txt', 'AA', config); + assert.ok(state['a.txt'].edits.length >= 1); + assert.strictEqual(state['b.txt'].edits.length, 0); + }); + + test('inserting multiple lines at beginning merges with existing hunk', function () { + const file = 'start.txt'; + const v0 = ['line1', 'line2', 'line3'].join('\n'); + let state: RecentEditMap = {}; + state = recentEditsReducer(state, file, v0, config); + // edit in the middle + const v1arr = ['line1', 'X', 'line2', 'line3']; + state = recentEditsReducer(state, file, v1arr.join('\n'), config); + assert.strictEqual(state[file].edits.length, 1); + // now insert two lines at the top + const v2arr = ['Y', 'Z', ...v1arr]; + state = recentEditsReducer(state, file, v2arr.join('\n'), config); + assert.strictEqual(state[file].edits.length, 1); + const { diff } = state[file].edits[0]; + assert.strictEqual(diff.pre, 0); + // first two added lines should be Y, Z + assert.deepStrictEqual(diff.added.slice(0, 2), ['Y', 'Z']); + // and the middle edit X should still be present + assert.ok(diff.added.includes('X')); + }); + + test('many sequential line inserts (aka typing) merge into one hunk', function () { + const file = 'typingtest.txt'; + let state: RecentEditMap = {}; + const fileLines = Array.from({ length: 100 }, (_, i) => `L${i + 1}`); + state = recentEditsReducer(state, file, fileLines.join('\n'), config); + + const whatToType = 'This is a multi-line bit of text.\nI sure hope everything works as planned.\nAnyways...'; + fileLines[50] = ''; + for (const char of whatToType) { + fileLines[50] += char; + state = recentEditsReducer(state, file, fileLines.join('\n'), config); + assert.strictEqual(state[file].edits.length, 1); + } + + // All inserts should collapse into a single hunk + assert.strictEqual(state[file].edits.length, 1); + const diff = unifiedDiff(state[file].edits[0].diff); + + assert.equal( + diff, + ` +--- a/typingtest.txt ++++ b/typingtest.txt +@@ -48,7 +48,9 @@ + L48 + L49 + L50 +-L51 ++This is a multi-line bit of text. ++I sure hope everything works as planned. ++Anyways... + L52 + L53 + L54 + ` + .replace(/\n {12}/g, '\n') + .trim() + '\n' + ); + }); + + test('huge change containing tiny change merges into one edit', function () { + const file = 'file.txt'; + let state: RecentEditMap = {}; + const fileLines = Array.from({ length: 100 }, (_, i) => `line ${i + 1}`); + state = recentEditsReducer(state, file, fileLines.join('\n'), config); + + for (let i = 30; i < 60; i++) { + fileLines[i] = `line ${i + 1} has changed`; + } + state = recentEditsReducer(state, file, fileLines.join('\n'), config); + assert.strictEqual(state[file].edits.length, 1); + + fileLines[50] = 'here comes another edit'; + state = recentEditsReducer(state, file, fileLines.join('\n'), config); + assert.strictEqual(state[file].edits.length, 1); + + assert.ok(state[file].edits[0].diff.added.includes('here comes another edit')); + assert.ok(state[file].edits[0].diff.removed.includes('line 51')); + }); + + test('two large overlapping changes merge into one edit', function () { + // deep copy + const configCopy = JSON.parse(JSON.stringify(config)) as RecentEditsConfig; + configCopy.maxCharsPerEdit = 10000; // Set a large enough limit to allow merging + const file = 'file.txt'; + let state: RecentEditMap = {}; + const fileLines = Array.from({ length: 100 }, (_, i) => `line ${i + 1}`); + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + + for (let i = 30; i < 60; i++) { + fileLines[i] = `line ${i + 1} has changed in the first edit`; + } + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + assert.strictEqual(state[file].edits.length, 1); + assert.equal(state[file].edits[0].diff.added.length, 30); + assert.equal(state[file].edits[0].diff.removed.length, 30); + + for (let i = 40; i < 80; i++) { + fileLines[i] = `line ${i + 1} has changed in the second edit`; + } + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + assert.strictEqual(state[file].edits.length, 1); + + assert.ok(state[file].edits[0].diff.added.includes('line 31 has changed in the first edit')); + assert.ok(state[file].edits[0].diff.added.includes('line 50 has changed in the second edit')); + assert.ok(state[file].edits[0].diff.removed.includes('line 50')); + assert.equal(state[file].edits[0].diff.added.length, 50); + assert.equal(state[file].edits[0].diff.removed.length, 50); + }); + + test('two large overlapping changes in reverse order merge into one edit', function () { + const configCopy = JSON.parse(JSON.stringify(config)) as RecentEditsConfig; + configCopy.maxCharsPerEdit = 10000; // Set a large enough limit to allow merging + const file = 'file.txt'; + let state: RecentEditMap = {}; + const fileLines = Array.from({ length: 100 }, (_, i) => `line ${i + 1}`); + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + + for (let i = 40; i < 80; i++) { + fileLines[i] = `line ${i + 1} has changed in the first edit`; + } + + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + assert.strictEqual(state[file].edits.length, 1); + assert.equal(state[file].edits[0].diff.added.length, 40); + assert.equal(state[file].edits[0].diff.removed.length, 40); + + for (let i = 30; i < 60; i++) { + fileLines[i] = `line ${i + 1} has changed in the second edit`; + } + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + assert.strictEqual(state[file].edits.length, 1); + + assert.ok(state[file].edits[0].diff.added.includes('line 31 has changed in the second edit')); + assert.ok(state[file].edits[0].diff.added.includes('line 50 has changed in the second edit')); + assert.ok(state[file].edits[0].diff.added.includes('line 70 has changed in the first edit')); + assert.ok(state[file].edits[0].diff.removed.includes('line 50')); + assert.equal(state[file].edits[0].diff.added.length, 50); + assert.equal(state[file].edits[0].diff.removed.length, 50); + }); + + test('edits larger than maxCharsPerEdit get removed', function () { + const configCopy = JSON.parse(JSON.stringify(config)) as RecentEditsConfig; + configCopy.maxCharsPerEdit = 100; + const file = 'file.txt'; + let state: RecentEditMap = {}; + const fileLines = Array.from({ length: 100 }, (_, i) => `line ${i + 1}`); + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + + for (let i = 40; i < 80; i++) { + fileLines[i] = `line ${i + 1} has changed in the first edit`; + } + state = recentEditsReducer(state, file, fileLines.join('\n'), configCopy); + + assert.equal(state[file].edits.length, 0); + }); +}); + +suite('getDiff', function () { + test('insertion builds DiffHunk correctly', function () { + const prev = ['1', '2']; + const next = ['1', 'X', '2']; + const span = { start: 1, endPrev: 1, endNew: 1 } as const; + const h = getDiff('file', prev, next, span.start, span.endPrev, span.endNew, 0); + assert.deepStrictEqual(h.removed, ['2']); + assert.deepStrictEqual(h.added, ['X']); + }); + + test('deletion builds DiffHunk correctly', function () { + const prev = ['1', 'X', '2']; + const next = ['1', '2']; + const span = { start: 1, endPrev: 1, endNew: 0 } as const; + const h = getDiff('file', prev, next, span.start, span.endPrev, span.endNew, 0); + assert.deepStrictEqual(h.removed, ['X']); + assert.deepStrictEqual(h.added, []); + }); + + test('replacement builds DiffHunk correctly', function () { + const prev = ['1', '2', '3']; + const next = ['1', 'different', '3']; + const span = { start: 1, endPrev: 1, endNew: 1 } as const; + const hunk = getDiff('file', prev, next, span.start, span.endPrev, span.endNew, 0); + assert.deepEqual(hunk, { + added: ['different'], + after: [], + before: [], + file: 'file', + post: 2, + pre: 1, + removed: ['2'], + }); + }); +}); + +suite('getAllRecentEditsByTimestamp', function () { + test('orders and slices correctly', function () { + const map: RecentEditMap = { + a: { + originalContent: '', + currentContent: '', + edits: [ + { + file: 'a', + startLine: 0, + endLine: 0, + diff: getDiff('a', [], [''], 0, 0, 0, 0), + timestamp: 5, + }, + ], + }, + b: { + originalContent: '', + currentContent: '', + edits: [ + { + file: 'b', + startLine: 0, + endLine: 0, + diff: getDiff('b', [], [''], 0, 0, 0, 0), + timestamp: 3, + }, + ], + }, + }; + const all = getAllRecentEditsByTimestamp(map); + assert.strictEqual(all[0].file, 'b'); + assert.strictEqual(all[1].file, 'a'); + }); +}); + +suite('buildIncomingEdit insertion at top', function () { + test('incoming hunk pre-populates pre/post and added lines', function () { + const prev = ['a', 'b']; + const next = ['X', 'a', 'b']; + const span = { start: 0, endPrev: 1, endNew: 0 } as const; + const h = buildIncomingEdit('f', prev, next, span, config); + assert.deepStrictEqual(h.diff.added, ['X']); + assert.strictEqual(h.diff.pre, 0); + }); +}); + +suite('trimOldFilesFromState', function () { + test('does nothing when modified files count is <= maxFiles', function () { + const baseDiff = getDiff('f', ['x'], ['y'], 0, 0, 0, 0); + const state: RecentEditMap = { + a: { originalContent: '', currentContent: '', edits: [] }, + b: { + originalContent: '', + currentContent: '', + edits: [{ file: 'b', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 1 }], + }, + }; + + const trimmed = trimOldFilesFromState(state, 2); + // 'a' has no edits, 'b' is within limit, both stay + assert.deepStrictEqual(Object.keys(trimmed).sort(), ['a', 'b']); + }); + + test('trims oldest modified files beyond maxFiles', function () { + const baseDiff = getDiff('f', ['x'], ['y'], 0, 0, 0, 0); + const state: RecentEditMap = { + one: { + originalContent: '', + currentContent: '', + edits: [{ file: 'one', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 10 }], + }, + two: { + originalContent: '', + currentContent: '', + edits: [{ file: 'two', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 20 }], + }, + three: { + originalContent: '', + currentContent: '', + edits: [{ file: 'three', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 30 }], + }, + }; + + const trimmed = trimOldFilesFromState(state, 2); + // Should drop 'one' (oldest), keep 'two' and 'three' + assert.deepStrictEqual(Object.keys(trimmed).sort(), ['three', 'two']); + // Ensure the entries for 'two' and 'three' remain intact + assert.strictEqual(trimmed.two.edits[0].timestamp, 20); + assert.strictEqual(trimmed.three.edits[0].timestamp, 30); + }); + + test('keeps unmodified files when trimming', function () { + const baseDiff = getDiff('f', ['x'], ['y'], 0, 0, 0, 0); + const state: RecentEditMap = { + unmodified: { + originalContent: '', + currentContent: '', + edits: [], + }, + old: { + originalContent: '', + currentContent: '', + edits: [{ file: 'old', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 1 }], + }, + mid: { + originalContent: '', + currentContent: '', + edits: [{ file: 'mid', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 2 }], + }, + recent: { + originalContent: '', + currentContent: '', + edits: [{ file: 'recent', startLine: 0, endLine: 0, diff: baseDiff, timestamp: 3 }], + }, + }; + + const trimmed = trimOldFilesFromState(state, 2); + // 'old' is the oldest modified and should be trimmed; + // 'mid' and 'recent' stay, and 'unmodified' (no edits) always stays + assert.deepStrictEqual(Object.keys(trimmed).sort(), ['mid', 'recent', 'unmodified']); + + // verify 'old' was removed + assert.strictEqual(trimmed.old, undefined); + // verify timestamps for the kept files + assert.strictEqual(trimmed.mid.edits[0].timestamp, 2); + assert.strictEqual(trimmed.recent.edits[0].timestamp, 3); + }); +}); + +function humanSize(bytes: number): string { + const sizes = ['b', 'kb', 'mb']; + let i = 0; + while (bytes >= 1024 && i < sizes.length - 1) { + bytes /= 1024; + i++; + } + return `${bytes}${sizes[i]}`; +} + +suite('recentEditsReducer performance', function () { + const initialFileContents = readFileSync(__filename, 'utf8'); + const complexityData: ComplexityData[] = []; + + // use no more than 20% of single core CPU time processing the input of the fastest typers + // this threshold needs to be pretty generous since this runs on tiny CI boxes + const fastTypingSpeedWPM = 100; + const averageWordLength = 5; + const fastTypingSpeedCPS = (fastTypingSpeedWPM / 60) * averageWordLength; // CPS means characters per second + const maxCPUTime = 0.2; + const minCPS = Math.ceil(fastTypingSpeedCPS * (1 / maxCPUTime)); + + this.retries(2); + + // 8kb-8mb + for (let fileSize = 8192; fileSize <= 8192 * 1024; fileSize *= 2) { + test(`random inserts in a ${humanSize(fileSize)} file`, function () { + // repeat the fileContents until they hit the desired size + let fileContents = initialFileContents + .repeat(Math.ceil(fileSize / initialFileContents.length)) + .slice(0, fileSize); + + const file = 'performantfile.txt'; + let state = recentEditsReducer({}, file, fileContents, config); + + const startTime = performance.now(); + let i = 0; + while (performance.now() - startTime < 500) { + const textToInsert = i % 10 === 0 ? '\n' : 'X'; + const randomPoint = Math.floor(Math.random() * fileContents.length); + fileContents = fileContents.slice(0, randomPoint) + textToInsert + fileContents.slice(randomPoint); + state = recentEditsReducer(state, file, fileContents, config); + i++; + } + const endTime = performance.now(); + const millisecondsPerCharacter = (endTime - startTime) / i; + const charactersPerSecond = i / ((endTime - startTime) / 1000); + + if (state[file]) { + // edits were tracked, log timing + complexityData.push({ + n: fileSize, + time: millisecondsPerCharacter, + }); + const cleanCharactersPerSecond = (Math.round(charactersPerSecond * 100) / 100).toFixed(2); + + assert.ok( + charactersPerSecond > minCPS, + `Edits per second (${cleanCharactersPerSecond}) must be at least ${minCPS}` + ); + } else { + console.warn( + `Warning: recentEditsReducer did not track edits for a ${humanSize(fileSize)} file. This may be due to the file being too large.` + ); + } + }); + } + + test('must have linear or sublinear time complexity', function () { + const { model } = determineTimeComplexity(complexityData); + assert.match( + model.type, + /^(sub)?linear$/, + `Time complexity must be linear or sublinear. Got ${model.name} which is ${model.type}` + ); + }); +}); + +suite('summarizeEdit function', function () { + test('return null if diff added and removed lines are all empty after stripping whitespace', function () { + const edit = { + startLine: 2, + endLine: 3, + diff: { + removed: [' ', '\t'], + added: [' ', '\n'], + } as DiffHunk, + } as RecentEdit; + const result = summarizeEdit(edit, config); + assert.strictEqual(result, null); + }); + test('return null if diff added or diff removed are over 100 lines', function () { + const edit = { + startLine: 2, + endLine: 3, + diff: { + removed: Array(101).fill('a') as string[], + added: Array(101).fill('b') as string[], + } as DiffHunk, + } as RecentEdit; + const configCopy = JSON.parse(JSON.stringify(config)) as RecentEditsConfig; + configCopy.maxLinesPerEdit = 100; + const result = summarizeEdit(edit, configCopy); + assert.strictEqual(result, null); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/render/renderNode.ts b/completions-sample-code/vscode-node/lib/src/prompt/render/renderNode.ts new file mode 100644 index 0000000..2c75959 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/render/renderNode.ts @@ -0,0 +1,357 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PriorityQueue } from '../../util/priorityQueue'; +import { DEFAULT_ELISION_MARKER, getAvailableNodeId, NodeCostFunction } from './utils'; + +export type NodeId = number; + +/** + * IVirtualNodes are abstract, potentially mutable nodes that can be snapshotted to + * concrete, immutable RenderNodes. Virtual nodes do not need to have all the information required + * for RenderNodes - for example, we don't need to know which cost function will be used when we + * construct a virtual node, and some aspects of their rendering (canMerge and elisionMarker) are + * optional. Virtual nodes can in principle also be useful in cases where the children are specified only implicitly, + * but have not yet been concretely initialized. + * + * Although virtual nodes are in principle mutable, they are EXPECTED to change their `id` if they + * are changed in other ways (for example, adding or removing children) + * + * The length of the text of a virtual node should be the length of its children, plus one. This + * is because the and the children of a virtual node interleave each other. For example: + * + * text: ['function hello_world() {\n\t', '\n\t', '\n}'] + * children: ['console.log('hello world');', '//another child'] + * + * The main example of a IVirtualNode (besides RenderNodes) right now is the ContextNode defined in #prompt/ast + */ +export interface IVirtualNode { + id: NodeId; + /** Text fragments that this node renders, excluding children. + * The length of this array must be equal to the number of children + 1. */ + text: readonly string[]; // length of children + 1 + /** Child nodes that this node renders, interleaved with text fragments. */ + children: readonly IVirtualNode[]; + canMerge?: boolean; + elisionMarker?: string; +} + +/** + * RenderNodes are frozen IVirtualNodes. In addition to the properties of IVirtualNodes, + * they have an associated cost (how expensive it is to render this node -- for example, its token length) + * and a weight (how desirable it is to render this node - higher weight indicates a greater preference + * to render it). + * + * RenderNodes contain the state required to recursively render a tree of nodes, within a given budget, + * replacing the least valuable nodes with elision markers. In addition, they offer a convenience method + * (`updateWeights`) for setting weights in the entire tree, and then "rectifying" the weights to minimally + * redistribute weight upwards in the tree so that parents are always more valuable than children. + * + * We don't use a class for RenderNodes because they get passed across thread boundaries. + */ +export interface RenderNode extends IVirtualNode { + readonly id: NodeId; + readonly text: readonly string[]; + readonly children: readonly RenderNode[]; + /** How much it costs to render this node, excluding children. Should be at least 1. */ + readonly cost: number; + /** How important this node is - higher means preferred for rendering. Should be nonnegative. */ + weight: number; + /** Weight, but rectified to ensure parent nodes are always more valuable than children. */ + rectifiedWeight?: number; + /** Whether this (elided) node can be merged with the previous (elided) children into a single elision marker. */ + canMerge: boolean; + /** The text to use when this node is elided. */ + elisionMarker: string; + /** Only render this node if a child is also rendered. Note that setting this + * to `true` on a leaf will prevent it from being rendered. */ + requireRenderedChild: boolean; +} + +export function createRenderNode(partial: Partial<RenderNode>): RenderNode { + const node: RenderNode = { + id: partial.id ?? getAvailableNodeId(), + text: partial.text ?? new Array((partial.children?.length ?? 0) + 1).fill(''), + children: partial.children ?? [], + cost: partial.cost ?? 1, + weight: partial.weight ?? 0, + rectifiedWeight: partial.rectifiedWeight, + canMerge: partial.canMerge ?? false, + elisionMarker: partial.elisionMarker ?? DEFAULT_ELISION_MARKER, + requireRenderedChild: partial.requireRenderedChild ?? false, + }; + if (node.text.length !== node.children.length + 1) { + throw new Error( + `RenderNode text length (${node.text.length}) must be children length + 1 (${node.children.length + 1})` + ); + } + return node; +} + +function isRenderedChildRequired(node: RenderNode): boolean { + return node.requireRenderedChild || (node.rectifiedWeight ?? node.weight) > node.weight; +} + +export function rectifiedValue(node: RenderNode): number { + return (node.rectifiedWeight ?? node.weight) / Math.max(node.cost, 1); +} + +/** + * Assign weights to nodes, while recursively minimally redistributing weights from children to ancestors + * so that the rectified value (rectifiedWeight / cost) of each node is no greater than the value of its parent. + * If no `weighter` is specified, uses the existing node weights a just redistributes from children to ancestors. + */ +export function rectifyWeights(node: RenderNode, weighter?: (node: RenderNode) => number) { + const rectificationQueue = recursivelyRectifyWeights(node, weighter); + for (const { item, priority } of rectificationQueue.clear()) { + for (const node of item.nodes) { + node.rectifiedWeight = priority * Math.max(node.cost, 1); + } + } +} + +type NodeGroup = { + nodes: RenderNode[]; + totalCost: number; + totalWeight: number; +}; +function recursivelyRectifyWeights( + node: RenderNode, + weighter?: (node: RenderNode) => number +): PriorityQueue<NodeGroup> { + const childQueues = node.children.map(child => recursivelyRectifyWeights(child, weighter)); + node.weight = Math.max(0, weighter ? weighter(node) : node.weight); + if (node.weight === 0 && childQueues.reduce((sum, q) => sum + q.size, 0) === 0) { + return new PriorityQueue<NodeGroup>([]); + } + + const merged: PriorityQueue<NodeGroup> = new PriorityQueue(childQueues.flatMap(queue => queue.clear())); + const group: NodeGroup = { + nodes: [node], + totalCost: node.cost, + totalWeight: node.weight, + }; + + // Combine with descendants until the combined average value is greater than or equal to the next item in the queue + while ((merged.peek()?.priority ?? 0) > group.totalWeight / Math.max(group.totalCost, 1)) { + const { item } = merged.pop()!; + group.nodes.push(...item.nodes); + group.totalCost += item.totalCost; + group.totalWeight += item.totalWeight; + } + merged.insert(group, group.totalWeight / Math.max(group.totalCost, 1)); + return merged; +} + +export type RenderedText = { + text: string; + cost: number; + renderedNodes: Map<NodeId, RenderNode>; +}; +type RenderOptions = Partial<{ + /* The maximum cost of the rendered text. If undefined, we render every unmasked node */ + budget: number; + /* A node or list of nodes to exclude from rendering (along with its children). */ + mask: NodeId | NodeId[]; + /* A true cost function to use when enforcing the budget. + * If omitted, the cost is the sum of the costs of the rendered nodes. + * + * This is used to strictly enforce a budget when the cost of concatenating nodes can exceed the sum of their costs, + * as in the case of tokenized lengths. In this case, multiple elision rounds may be required as we use the + * nodewise costs to estimate how many marginal nodes need to be removed to satisfy the overall budget. + */ + costFunction: (text: string) => number; +}>; + +/** + * Recursively render this node and its children + * + * @return An object containing the rendered text and its cost, which will either be the length of the text + * or the result of the cost function if provided. + */ +export function render(node: RenderNode, options: RenderOptions = {}): RenderedText { + const { budget, mask, costFunction } = options; + const exclude = mask ?? []; + const exclusionSet = new Set(Array.isArray(exclude) ? exclude : [exclude]); + + if ((budget ?? node.cost) < node.cost || exclusionSet.has(node.id)) { + return { + text: node.elisionMarker, + cost: costFunction ? costFunction(node.elisionMarker) : node.elisionMarker.length, + renderedNodes: new Map(), + }; + } + + if (budget === undefined) { + // just elide any excluded nodes (and their descendants) + const elider = (node: RenderNode) => exclusionSet.has(node.id); + const renderParts: string[] = []; + const renderedNodes: Map<NodeId, RenderNode> = new Map(); + recursivelyRender(node, renderParts, elider, renderedNodes); + if (renderParts.length === 0) { + return renderEmpty(node, costFunction); + } + const text = renderParts.join(''); + const cost = costFunction + ? costFunction(text) + : [...renderedNodes.values()].reduce((sum, n) => sum + n.cost, 0); + return { text, cost, renderedNodes }; + } + + // Elide nodes that are not in the rendered set + let targetNodes = new Map<NodeId, RenderNode>(); + // With the additional cost function, we keep track of the order in which we select nodes for rendering + // This is used to remove nodes that are marginally valuable if the final true cost exceeds the budget + const marginalNodes: RenderNode[] = []; + // Include highest-value non-excluded nodes up to the budget + const explorationQueue = new PriorityQueue<RenderNode>([{ item: node, priority: rectifiedValue(node) }]); + let remainingBudget = budget; + while (remainingBudget > 0 && explorationQueue.size > 0) { + const { item } = explorationQueue.pop()!; + if (exclusionSet.has(item.id)) { + continue; + } + if (item.cost <= remainingBudget) { + remainingBudget -= item.cost; + targetNodes.set(item.id, item); + marginalNodes.push(item); + // Add children to the queue, prioritizing those with higher value + for (const child of item.children) { + explorationQueue.insert(child, rectifiedValue(child)); + } + } + } + + // We have a rendering plan that is projected to be within budget, but actual cost of the combined text may differ + // If we have a cost function, we may still need to iteratively remove nodes until the true cost is within budget + while (targetNodes.size > 0) { + const renderParts: string[] = []; + const elider = (node: RenderNode) => !targetNodes.has(node.id); + // `renderedNodes` will be a subset of `targetNodes`; some additional nodes may be elided due to + // the requirement to render at least one child + const renderedNodes = new Map<NodeId, RenderNode>(); + recursivelyRender(node, renderParts, elider, renderedNodes); + if (renderParts.length === 0) { + // If we didn't render anything, we can return the elision marker + return renderEmpty(node, costFunction); + } + const text = renderParts.join(''); + if (costFunction === undefined) { + // Within budget by construction + const cost = [...renderedNodes.values()].reduce((sum, n) => sum + n.cost, 0); + return { text, cost, renderedNodes }; + } + + let cost = costFunction(text); + if (cost <= budget) { + // If the cost of the rendered text is within budget, return it + return { text, cost, renderedNodes }; + } + + // Otherwise, we will elide additional nodes and try again + targetNodes = renderedNodes; + while (marginalNodes.length > 0 && cost > budget) { + const node = marginalNodes.pop()!; + if (targetNodes.has(node.id)) { + cost -= node.cost; // Use nodewise cost to *estimate* change in overall cost + targetNodes.delete(node.id); + } // Otherwise, we didn't render it because of requireRenderedChild + } + + if (marginalNodes.length === 0) { + // infeasible budget + break; + } + } + return renderEmpty(node, costFunction); +} + +function renderEmpty(node: RenderNode, costFunction?: (text: string) => number): RenderedText { + return { + text: node.elisionMarker, + cost: costFunction ? costFunction(node.elisionMarker) : node.elisionMarker.length, + renderedNodes: new Map(), + }; +} + +function recursivelyRender( + node: RenderNode, + parts: string[], + elider: (node: RenderNode) => boolean, + renderedNodes: Map<NodeId, RenderNode>, + mergeElision: boolean = false +): boolean { + const numParts = parts.length; + if (elider(node)) { + if (numParts >= 2) { + if ( + mergeElision || + (parts[numParts - 2] === node.elisionMarker && parts[numParts - 1].trim().length === 0) + ) { + parts.pop(); // elide by removing separator from previous elision + return false; + } + } + parts.push(node.elisionMarker); + return false; + } + + // Combine text fragments and rendered children + let requiresChild = isRenderedChildRequired(node); + let didRender = true; + for (const [i, child] of node.children.entries()) { + parts.push(node.text[i] ?? ''); + didRender = recursivelyRender(child, parts, elider, renderedNodes, child.canMerge && !didRender); + requiresChild &&= !didRender; + } + if (requiresChild) { + // We did not render any child, but are required to render one + // Revert `parts` to its state before this node's text fragments + while (parts.length > numParts) { + parts.pop(); + } + return false; + } + // Finish rendering this node with the last text fragment + parts.push(node.text[node.text.length - 1] ?? ''); + renderedNodes.set(node.id, node); + return true; +} + +/** + * Freeze a tree of virtual nodes into RenderNodes, using a given cost function and elision marker. + * + * Optionally, make use a cache (such as an LRUCacheMap) mapping IDs to RenderNodes. When we encounter a cached ID, + * we return the cached RenderNode without recursing further. For this to behave as expected, the IVirtualNodes + * *must* change their ID whenever their subtree changes. + */ +export function snapshot( + node: IVirtualNode, + costFunction: NodeCostFunction, + elisionMarker: string = DEFAULT_ELISION_MARKER +): RenderNode { + const children = node.children.map(child => snapshot(child, costFunction, elisionMarker)); + elisionMarker = node.elisionMarker ?? elisionMarker; + const cost = costFunction(node); + const renderNode: RenderNode = createRenderNode({ + ...node, + children, + cost, + weight: 0, + elisionMarker: node.elisionMarker ?? elisionMarker, + }); + return renderNode; +} + +export const EMPTY_NODE: RenderNode = { + id: getAvailableNodeId(), + text: [''], + children: [], + cost: 0, + weight: 0, + elisionMarker: '', + canMerge: true, + requireRenderedChild: false, +}; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/render/test/renderNode.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/render/test/renderNode.test.ts new file mode 100644 index 0000000..1e890c2 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/render/test/renderNode.test.ts @@ -0,0 +1,127 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { createRenderNode, rectifiedValue, rectifyWeights, render } from '../renderNode'; +import { DEFAULT_ELISION_MARKER } from '../utils'; + +suite('RenderNode', function () { + test('constructs node without children', function () { + const node = createRenderNode({ text: ['a'], cost: 0, weight: 5 }); + assert.deepEqual(node.text, ['a']); + assert.deepEqual(node.children, []); + assert.deepStrictEqual(node.cost, 0); + assert.deepStrictEqual(node.weight, 5); + assert.deepStrictEqual(node.elisionMarker, DEFAULT_ELISION_MARKER); + }); + + test('constructs node with children', function () { + const child = createRenderNode({ text: ['c'], cost: 1, weight: 3 }); + const node = createRenderNode({ text: ['a', 'b'], children: [child], cost: 2, weight: 5 }); + assert.deepEqual(node.children.length, 1); + }); + + test('should check that text is children + 1', function () { + assert.throws(() => createRenderNode({ text: ['a', 'b'], children: [], cost: 2, weight: 5 })); + }); + + test('renders all nodes without budget', function () { + const child = createRenderNode({ text: ['b'], cost: 1, weight: 2 }); + const node = createRenderNode({ text: ['a', 'c'], children: [child], cost: 2, weight: 5 }); + const result = render(node); + assert.deepStrictEqual(result.text, 'abc'); + assert.deepStrictEqual(result.cost, 3); + }); + + test('renders with budget, elides child if over budget', function () { + const child = createRenderNode({ text: ['bb'], cost: 2, weight: 2 }); + const node = createRenderNode({ text: ['aa', 'cc'], children: [child], cost: 4, weight: 5 }); + // Budget only enough for parent + const result = render(node, { budget: 5 }); + assert.deepStrictEqual(result.text, `aa${child.elisionMarker}cc`); + assert.deepStrictEqual(result.cost, 4); + }); + + test('renders with exclude', function () { + const child = createRenderNode({ text: ['bb'], cost: 2, weight: 2 }); + const node = createRenderNode({ text: ['aa', 'cc'], children: [child], cost: 4, weight: 5 }); + assert.deepStrictEqual(render(node, { mask: node.id }).text, node.elisionMarker); + assert.deepStrictEqual(render(node, { mask: child.id }).text, `aa${child.elisionMarker}cc`); + }); + + test('canMerge merges adjacent elided children into a single elision marker', function () { + // Create child nodes, some of which will be excluded (elided) + const child1 = createRenderNode({ text: ['A'], cost: 1, weight: 1 }); + const child2 = createRenderNode({ text: ['B'], cost: 1, weight: 1 }); + const child2Merge = createRenderNode({ text: ['B'], cost: 1, weight: 1, canMerge: true }); + const child3 = createRenderNode({ text: ['C'], cost: 1, weight: 1 }); + + const nodeWithoutMerge = createRenderNode({ + text: ['(', ',', ',', ')'], + children: [child1, child2, child3], + cost: 1, + weight: 1, + }); + const nodeWithMerge = createRenderNode({ + text: ['(', ',', ',', ')'], + children: [child1, child2Merge, child3], + cost: 1, + weight: 1, + }); + + assert.deepStrictEqual(render(nodeWithoutMerge, { mask: [child1.id, child2.id] }).text, '([...],[...],C)'); + assert.deepStrictEqual(render(nodeWithMerge, { mask: [child1.id, child2Merge.id] }).text, '([...],C)'); + }); + + test('renders with multiple children, one over budget', function () { + const child1 = createRenderNode({ text: ['bb'], cost: 2, weight: 3 }); + const child2 = createRenderNode({ text: ['dd'], cost: 2, weight: 2 }); + const node = createRenderNode({ text: ['aa', 'cc', 'ee'], children: [child1, child2], cost: 6, weight: 6 }); + // Budget only enough for parent and one child + assert.deepStrictEqual(render(node, { budget: 8 }).text, `aabbcc${child2.elisionMarker}ee`); + }); + + test('renders with custom costFunction', function () { + // Use a custom elision marker since it's now counted in the cost + const child1 = createRenderNode({ text: ['bb'], cost: 2, weight: 2, elisionMarker: '.' }); + const child2 = createRenderNode({ text: ['dd'], cost: 2, weight: 3, elisionMarker: '.' }); + const node = createRenderNode({ + text: ['aa', 'cc', 'ee'], + children: [child1, child2], + cost: 6, + weight: 6, + elisionMarker: '.', + }); + // The second child doesn't fit anymore, since now the cost is based on length + // and so the markers also have a cost. + assert.deepStrictEqual(render(node, { budget: 8, costFunction: t => t.length }).text, 'aa.cc.ee'); + }); + + test('infeasible budget returns elision marker', function () { + const node = createRenderNode({ text: ['aa'], cost: 2, weight: 5 }); + const result = render(node, { budget: 1 }); + assert.deepStrictEqual(result.text, node.elisionMarker); + assert.deepStrictEqual(result.cost, 5); + }); + + test('redistributes weights (default weighter)', function () { + const child1 = createRenderNode({ text: ['d'], cost: 1, weight: 5 }); + const child2 = createRenderNode({ text: ['e'], cost: 1, weight: 5 }); + const root = createRenderNode({ text: ['a', 'b', 'c'], children: [child1, child2], cost: 3, weight: 2 }); + + rectifyWeights(root); + assert.ok(root.children.every(child => rectifiedValue(child) <= rectifiedValue(root))); + }); + + test('requireRenderedChild after redestributing weight', function () { + const child1 = createRenderNode({ text: ['d'], cost: 1, weight: 5 }); + const child2 = createRenderNode({ text: ['e'], cost: 1, weight: 5 }); + const root = createRenderNode({ text: ['a', 'b', 'c'], children: [child1, child2], cost: 3, weight: 2 }); + + assert.deepStrictEqual(render(root, { budget: 3 }).text, 'a[...]b[...]c'); + rectifyWeights(root); + assert.deepStrictEqual(render(root, { budget: 3 }).text, '[...]'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/render/utils.ts b/completions-sample-code/vscode-node/lib/src/prompt/render/utils.ts new file mode 100644 index 0000000..aae0b71 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/render/utils.ts @@ -0,0 +1,16 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IVirtualNode, NodeId } from './renderNode'; + +export const DEFAULT_ELISION_MARKER = '[...]'; + +let nextNodeId: NodeId = 0; +export function getAvailableNodeId(): NodeId { + return nextNodeId++; +} + +export type NodeCostFunction = (node: IVirtualNode) => number; + diff --git a/completions-sample-code/vscode-node/lib/src/prompt/repository.ts b/completions-sample-code/vscode-node/lib/src/prompt/repository.ts new file mode 100644 index 0000000..cfa1025 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/repository.ts @@ -0,0 +1,272 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { AdoRepoId, getAdoRepoIdFromFetchUrl, getGithubRepoIdFromFetchUrl, GithubRepoId, parseRemoteUrl } from '../../../../../../platform/git/common/gitService'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { FileIdentifier, ICompletionsFileSystemService } from '../fileSystem'; +import { LRUCacheMap } from '../helpers/cache'; +import { dirname, getFsUri, joinPath } from '../util/uri'; + +interface RepoInfo { + /** + * the parent directory of .git as a URI, or "" if there is no .git directory + */ + baseFolder: { readonly uri: string }; + /** + * the full url of the remote origin, e.g. git@github.com:github/synth.git or https://github.com/microsoft/vscode-copilot-chat.git + */ + url: string; + /** + * the hostname of the remote repository, e.g. github.com + */ + hostname: string; + /** + * Git remote pathname + */ + pathname: string; + /** + * Data for github.com and ADO repositories. + */ + repoId: GithubRepoId | AdoRepoId | undefined; +} + +export type MaybeRepoInfo = RepoInfo | undefined | ComputationStatus; + +export function tryGetGitHubNWO(repoInfo: MaybeRepoInfo): string | undefined { + if (repoInfo === undefined) { + return undefined; + } + if (repoInfo === ComputationStatus.PENDING) { + return undefined; + } + if (repoInfo.repoId?.type === 'github') { + return (repoInfo.repoId.org + '/' + repoInfo.repoId.repo).toLowerCase(); + } + return undefined; +} + +/** + * Sends off a computation to extract information about which git repo the file belongs to in the background. + * @param fileUri URI of a file under the repo + * @returns + * - If the computation is still running (in particular for the first time), returns ComputationStatus.PENDING. + * - If a file from this path has been looked at before, and no repository has been identified, returns undefined. + * - If a file from this path has been looked at before, and a repository has been identified, returns the repo info. + */ +export function extractRepoInfoInBackground(accessor: ServicesAccessor, uri: FileIdentifier): MaybeRepoInfo { + const baseFolder = dirname(uri); + return backgroundRepoInfo(accessor, baseFolder); +} + +// Note that we assume that the same filesystem path always returns the same repository information. +// If this changes on disk, or if two different contexts with different FileSystem implementations +// are passed for the same path, such as for a test, then the cached value may be incorrect +const backgroundRepoInfo = computeInBackgroundAndMemoize<RepoInfo | undefined, [FileIdentifier]>( + extractRepoInfo, + 10000 +); + +/** + * If the file is part of a git repository, return the information about the repository. + * @param uri URI of a folder or file in the repository + * @param fs The file system to be used + * @returns A RepoInfo object, or undefined if the file is not part of a git repository. + * If it does appear to be part of a git repository, but its information is not parsable, + * it returns a RepoInfo object with hostname, user and repo set to "". + */ +export async function extractRepoInfo(accessor: ServicesAccessor, uri: FileIdentifier): Promise<RepoInfo | undefined> { + const fs = accessor.get(ICompletionsFileSystemService); + + const fsUri = getFsUri(uri); + if (!fsUri) { return undefined; } + + const baseUri = await getRepoBaseUri(fs, fsUri); + if (!baseUri) { + return undefined; + } + const configUri = joinPath(baseUri, '.git', 'config'); + let gitConfig; + try { + gitConfig = await fs.readFileString(configUri); + } catch (_) { + // typically an ENOENT or EPERM, wrapped in varying ways depending on which FileSystem implementation is used + return undefined; + } + const url = getRepoUrlFromConfigText(gitConfig) ?? ''; + const parsedResult = parseRepoUrl(url); + const baseFolder = { uri: baseUri }; + if (parsedResult === undefined) { + return { baseFolder, url, hostname: '', pathname: '', repoId: undefined }; + } else { + return { baseFolder, url, hostname: parsedResult.host, pathname: parsedResult.path, repoId: parsedResult.repoId }; + } +} + +function parseRepoUrl( + url: string +): { host: string; path: string; repoId: GithubRepoId | AdoRepoId | undefined } | undefined { + const res = parseRemoteUrl(url); + if (!res) { + return undefined; + } + const repoId = getGithubRepoIdFromFetchUrl(url) ?? getAdoRepoIdFromFetchUrl(url); + return { ...res, repoId }; +} + +/** + * Returns the base folder of the git repository containing the file, or undefined if none is found. + * Will search recursively for a .git folder containing a config file. + */ +async function getRepoBaseUri(fileSystemService: ICompletionsFileSystemService, uri: string): Promise<string | undefined> { + // to make sure the while loop terminates, we make sure the path variable decreases in length + let previousUri = uri + '_add_to_make_longer'; + while (uri !== 'file:///' && uri.length < previousUri.length) { + const configUri = joinPath(uri, '.git', 'config'); + let result = false; + + try { + await fileSystemService.stat(configUri); + result = true; + } catch (reason) { + result = false; + } + + if (result) { + return uri; + } else { + previousUri = uri; + uri = dirname(uri); + } + } + return undefined; +} + +/** + * Parses a git config file, returning + * 1. remote.origin.url if it exists, + * 2. any remote.[name].url if it exists but not 1., + * 3. undefined if neither exist. + * Will throw if the file does not exist. + * + * The config format is expected to follow https://git-scm.com/docs/git-config#_configuration_file + * e.g. it could include lines like + [remote "origin"] + url = git@github.com:microsoft/vscode-copilot-chat.git + fetch = +refs/heads/*:refs/remotes/origin/* + * + * Known limitations: + * - This will not respect include and includeIf directions + * + * @param gitConfig the contents of the git config file + * @returns the url, or undefined if none found + */ +function getRepoUrlFromConfigText(gitConfig: string): string | undefined { + // We're looking for [remote "origin"] and [remote "name"] sections + + // section headers must be one line, + // except for whitespace, they're [section "subsection"] + // where subsection can contain " by escaping \" and + // can escape \ by \\ (so that e.g. it can be the last character before the ") + const remoteSectionRegex = /^\s*\[\s*remote\s+"((\\\\|\\"|[^\\"])+)"/; + // deprecated syntax: [section.subsection] + const deprecatedRemoteSectionRegex = /^\s*\[remote.([^"\s]+)/; + // extract the name of the remote -- assume it doesn't contain whitespace, and remember # and ; start comments + const setUrlRegex = /^\s*url\s*=\s*([^\s#;]+)/; + // use the following to check whether the current section ended + const newSectionRegex = /^\s*\[/; + + let remoteUrl: string | undefined = undefined; + let remoteSection = undefined; + let isWithinMultilineUrl = false; + for (const line of gitConfig.split('\n')) { + if (isWithinMultilineUrl && remoteUrl !== undefined) { + remoteUrl += line; + if (line.endsWith('\\')) { + remoteUrl = remoteUrl.substring(0, remoteUrl.length - 1); + } else { + isWithinMultilineUrl = false; + if (remoteSection === 'origin') { + // we're already finished + return remoteUrl; + } + } + } else { + // check whether a new section starts + const remoteSectionMatch = line.match(remoteSectionRegex) ?? line.match(deprecatedRemoteSectionRegex); + if (remoteSectionMatch) { + remoteSection = remoteSectionMatch[1]; + } else if (line.match(newSectionRegex)) { + remoteSection = undefined; + } else if (remoteUrl && remoteSection !== 'origin') { + // if we already have any remote url, only "origin" is more interesting + continue; + } else { + const urlMatch = line.match(setUrlRegex); + if (urlMatch) { + remoteUrl = urlMatch[1]; + if (remoteUrl.endsWith('\\')) { + remoteUrl = remoteUrl.substring(0, remoteUrl.length - 1); + isWithinMultilineUrl = true; + } else if (remoteSection === 'origin') { + // we're already finished + return remoteUrl; + } + } + } + } + } + return remoteUrl; +} + +/** + * Helper functionality for doing the computation in the background + */ + +export enum ComputationStatus { + PENDING, +} + +class CompletedComputation<T> { + readonly result: T; + constructor(result: T) { + this.result = result; + } +} + +/** + * Function wrapper that memoizes a given function to be computed in the background. + * Until the first computation is complete, the wrapper returns ComputationStatus.PENDING. + * The context is not taken into account for computing the cache key so the function may + * behave incorrectly if called with different contexts. + * @param fct A function returning a promise whose arguments are amenable to JSON.stringify. + * @param cacheSize Number of elements to cache. + * @returns The memoized function, which returns ComputationStatus.PENDING until the first computation is complete. + */ +function computeInBackgroundAndMemoize<S, P extends unknown[]>( + fct: (accessor: ServicesAccessor, ...args: P) => Promise<S>, + cacheSize: number +): (accessor: ServicesAccessor, ...args: P) => S | ComputationStatus { + const resultsCache = new LRUCacheMap<string, CompletedComputation<S>>(cacheSize); + const inComputation: Set<string> = new Set(); + return (accessor: ServicesAccessor, ...args: P) => { + const key = JSON.stringify(args); + const memorizedComputation = resultsCache.get(key); + if (memorizedComputation) { + return memorizedComputation.result; + } + if (inComputation.has(key)) { + // already being computed from a different call + return ComputationStatus.PENDING; + } + const computation = fct(accessor, ...args); + inComputation.add(key); + void computation.then(computedResult => { + // remove from inComputation + resultsCache.set(key, new CompletedComputation(computedResult)); + inComputation.delete(key); + }); + return ComputationStatus.PENDING; + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/compositeRelatedFilesProvider.ts b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/compositeRelatedFilesProvider.ts new file mode 100644 index 0000000..31d0746 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/compositeRelatedFilesProvider.ts @@ -0,0 +1,173 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IIgnoreService } from '../../../../../../../platform/ignore/common/ignoreService'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CancellationToken as ICancellationToken } from '../../../../types/src'; +import { ConfigKey, getConfig } from '../../config'; +import { ICompletionsFeaturesService } from '../../experiments/featuresService'; +import { ICompletionsFileSystemService } from '../../fileSystem'; +import { ICompletionsLogTargetService } from '../../logger'; +import { TelemetryWithExp } from '../../telemetry'; +import { NeighboringFileType } from './neighborFiles'; +import { + EmptyRelatedFilesResponse, + RelatedFilesDocumentInfo, + RelatedFilesProvider, + RelatedFilesResponse, + relatedFilesLogger, +} from './relatedFiles'; + +const cppLanguageIds = ['cpp', 'c', 'cuda-cpp']; +const typescriptLanguageIds = ['typescript', 'javascript', 'typescriptreact', 'javascriptreact']; +const csharpLanguageIds = ['csharp']; +const neighborFileTypeMap = new Map<string, NeighboringFileType>([ + ...cppLanguageIds.map(id => [id, NeighboringFileType.RelatedCpp] as const), + ...typescriptLanguageIds.map(id => [id, NeighboringFileType.RelatedTypeScript] as const), + ...csharpLanguageIds.map(id => [id, NeighboringFileType.RelatedCSharpRoslyn] as const), +]); + +function getNeighboringFileType(languageId: string): NeighboringFileType { + return neighborFileTypeMap.get(languageId) ?? NeighboringFileType.RelatedOther; +} + +export type ProviderCallback = ( + uri: string, + context: { flags: Record<string, unknown> }, + cancellationToken: ICancellationToken +) => Promise<RelatedFilesResponse | undefined>; + +type Provider = { + languageId: string; + extensionId: string; + callback: ProviderCallback; +}; + +export class CompositeRelatedFilesProvider extends RelatedFilesProvider { + protected providers: Map<string, Map<string, Provider>> = new Map(); + protected telemetrySent = false; + private reportedUnknownProviders = new Set<string>(); + constructor( + @IInstantiationService instantiationService: IInstantiationService, + @IIgnoreService ignoreService: IIgnoreService, + @ICompletionsFeaturesService private featuresService: ICompletionsFeaturesService, + @ICompletionsLogTargetService logTarget: ICompletionsLogTargetService, + @ICompletionsFileSystemService fileSystemService: ICompletionsFileSystemService, + ) { + super(instantiationService, ignoreService, logTarget, fileSystemService); + } + override async getRelatedFilesResponse( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined + ) { + const startTime = Date.now(); + const languageId = docInfo.clientLanguageId.toLowerCase(); + const fileType = getNeighboringFileType(languageId); + if (fileType === NeighboringFileType.RelatedOther && !this.reportedUnknownProviders.has(languageId)) { + this.reportedUnknownProviders.add(languageId); + relatedFilesLogger.warn(this.logTarget, `unknown language ${languageId}`); + } + this.relatedFilesTelemetry(telemetryData); + + relatedFilesLogger.debug(this.logTarget, `Fetching related files for ${docInfo.uri}`); + if (!this.isActive(languageId, telemetryData)) { + relatedFilesLogger.debug(this.logTarget, 'language-server related-files experiment is not active.'); + return EmptyRelatedFilesResponse; + } + + const languageProviders = this.providers.get(languageId); + if (!languageProviders) { + return EmptyRelatedFilesResponse; + } + try { + return this.convert(docInfo.uri, languageProviders, startTime, telemetryData, cancellationToken); + } catch (error) { + // When the command returns an empty std::optional, we get an Error exception with message: + // "Received message which is neither a response nor a notification message: {"jsonrpc": "2.0","id": 22}" + this.relatedFileNonresponseTelemetry(languageId, telemetryData); + // Return undefined to inform the caller that the command failed. + return undefined; + } + } + async convert( + uri: string, + providers: Map<string, Provider>, + startTime: number, + telemetryData: TelemetryWithExp, + token: ICancellationToken | undefined + ): Promise<RelatedFilesResponse | undefined> { + if (!token) { + token = { + isCancellationRequested: false, + onCancellationRequested: () => ({ dispose() { } }), + }; + } + const combined: RelatedFilesResponse = { entries: [], traits: [] }; + let allProvidersReturnedUndefined: boolean = providers.size > 0; + for (const provider of providers.values()) { + const response = await provider.callback(uri, { flags: {} }, token); + if (response) { + allProvidersReturnedUndefined = false; + combined.entries.push(...response.entries); + if (response.traits) { + combined.traits!.push(...response.traits); + } + for (const entry of response.entries) { + for (const uri of entry.uris) { + relatedFilesLogger.debug(this.logTarget, uri.toString()); + } + } + } + } + this.performanceTelemetry(Date.now() - startTime, telemetryData); + return allProvidersReturnedUndefined ? undefined : combined; + } + registerRelatedFilesProvider(extensionId: string, languageId: string, provider: ProviderCallback) { + const languageProvider = this.providers.get(languageId); + if (languageProvider) { + languageProvider.set(extensionId, { extensionId, languageId, callback: provider }); + } else { + this.providers.set(languageId, new Map([[extensionId, { extensionId, languageId, callback: provider }]])); + } + } + unregisterRelatedFilesProvider(extensionId: string, languageId: string, callback: ProviderCallback) { + const languageProvider = this.providers.get(languageId); + if (languageProvider) { + const currentProvider = languageProvider.get(extensionId); + if (currentProvider && currentProvider.callback === callback) { + languageProvider.delete(extensionId); + } + } + } + /** + * Providers should manage their own telemetry. + * These four methods are for backward compatibility with the C++ provider. + */ + isActive(languageId: string, telemetryData: TelemetryWithExp): boolean { + if (csharpLanguageIds.includes(languageId)) { + return ( + this.featuresService.relatedFilesVSCodeCSharp(telemetryData) || + this.instantiationService.invokeFunction(getConfig<boolean>, ConfigKey.RelatedFilesVSCodeCSharp) + ); + } else if (typescriptLanguageIds.includes(languageId)) { + return ( + this.featuresService.relatedFilesVSCodeTypeScript(telemetryData) || + this.instantiationService.invokeFunction(getConfig<boolean>, ConfigKey.RelatedFilesVSCodeTypeScript) + ); + } else if (cppLanguageIds.includes(languageId)) { + return ( + this.featuresService.cppHeadersEnableSwitch(telemetryData) + ); + } + return ( + this.featuresService.relatedFilesVSCode(telemetryData) || + this.instantiationService.invokeFunction(getConfig<boolean>, ConfigKey.RelatedFilesVSCode) + ); + } + relatedFilesTelemetry(telemetryData: TelemetryWithExp) { } + relatedFileNonresponseTelemetry(language: string, telemetryData: TelemetryWithExp) { } + performanceTelemetry(duration: number, telemetryData: TelemetryWithExp) { } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/neighborFiles.ts b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/neighborFiles.ts new file mode 100644 index 0000000..04c51c7 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/neighborFiles.ts @@ -0,0 +1,192 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { normalizeLanguageId, SimilarFileInfo } from '../../../../prompt/src/prompt'; +import { CancellationToken as ICancellationToken } from '../../../../types/src'; +import { ICompletionsFeaturesService } from '../../experiments/featuresService'; +import { ICompletionsLogTargetService } from '../../logger'; +import { TelemetryWithExp } from '../../telemetry'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { OpenTabFiles } from './openTabFiles'; +import { getRelatedFilesAndTraits, relatedFilesLogger, RelatedFileTrait } from './relatedFiles'; + +// There is a limitation of the number of the neighbor files. So I use the next strategies to pick the most relevant cursor focused files. +export enum NeighboringFileType { + None = 'none', // Do not add neighbor files. + OpenTabs = 'opentabs', // Add open files. + CursorMostRecent = 'cursormostrecent', // Add the most recent cursor focused files. + CursorMostCount = 'cursormostcount', // Add the most cursor focused files. + WorkspaceSharingSameFolder = 'workspacesharingsamefolder', // Add the workspace files sharing the same folder with the target file. + WorkspaceSmallestPathDist = 'workspacesmallestpathdist', // Add the workspace files according to their path distance toward the target file + OpenTabsAndCocommitted = 'opentabsandcocommitted', // Add open files and the co-committed files. + RelatedCSharp = 'related/csharp', // The Semantic Code Context says this file is related. + RelatedCSharpRoslyn = 'related/csharproslyn', // The C# language service says this file is related. + RelatedCpp = 'related/cpp', // The C++ language service says this file is related. + RelatedTypeScript = 'related/typescript', // The Typescript language service says this file is related. + RelatedCppSemanticCodeContext = 'related/cppsemanticcodecontext', // The Semantic Code Context says this file is related. + RelatedOther = 'related/other', // An unknown language service says this file is related. +} + +/** + * We found out that considering + * all **open** neighbor files (independent of the language) was not helpful. However, some + * specific languages (e.g. frontend frameworks) benefit from this approach. Leaving this + * function here for future reference, in case we want to experiment this approach again for + * specific languages that always use cross-language files. + * + * @param languageId Language ID of the current file + * @param neighborLanguageId Language ID of the neighbor file + * @returns Boolean value indicating whether the neighbor file should be considered + * (currently matching the current file's language with neighbors') + */ +export function considerNeighborFile(languageId: string, neighborLanguageId: string): boolean { + return normalizeLanguageId(languageId) === normalizeLanguageId(neighborLanguageId); +} + +export type NeighborsCollection = Map<string, SimilarFileInfo>; + +export interface INeighborSource { + getNeighborFiles( + uri: string, + languageId: string, + maxNumNeighborFiles: number + ): Promise<{ docs: NeighborsCollection; neighborSource: Map<NeighboringFileType, string[]> }>; +} + +export class NeighborSource { + // Limit the amount of neighbor data to pass to promptlib. + static MAX_NEIGHBOR_AGGREGATE_LENGTH = 200000; + static MAX_NEIGHBOR_FILES = 20; + + static EXCLUDED_NEIGHBORS = ['node_modules', 'dist', 'site-packages']; + + static defaultEmptyResult() { + return { + docs: new Map<string, SimilarFileInfo>(), + neighborSource: new Map<NeighboringFileType, string[]>(), + traits: [] as RelatedFileTrait[], + }; + } + + private static instance: INeighborSource | undefined; + + /** Reset the singleton instance for unit test only */ + static reset(): void { + NeighborSource.instance = undefined; + } + + static async getNeighborFilesAndTraits( + accessor: ServicesAccessor, + uri: string, + fileType: string, + telemetryData: TelemetryWithExp, + cancellationToken?: ICancellationToken, + data?: unknown, + forceRelatedFilesComputation?: boolean + ): Promise<{ + docs: NeighborsCollection; + neighborSource: Map<NeighboringFileType, string[]>; + traits: RelatedFileTrait[]; + }> { + const featuresService = accessor.get(ICompletionsFeaturesService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const instantiationService = accessor.get(IInstantiationService); + const docManager = accessor.get(ICompletionsTextDocumentManagerService); + if (NeighborSource.instance === undefined) { + NeighborSource.instance = instantiationService.createInstance(OpenTabFiles); + } + + const result = { + ...(await NeighborSource.instance.getNeighborFiles(uri, fileType, NeighborSource.MAX_NEIGHBOR_FILES)), + traits: [] as RelatedFileTrait[], + }; + + if (featuresService.excludeRelatedFiles(fileType, telemetryData)) { return result; } + + const doc = await docManager.getTextDocument({ uri }); + if (!doc) { + relatedFilesLogger.debug(logTarget, + 'neighborFiles.getNeighborFilesAndTraits', + `Failed to get the related files: failed to get the document ${uri}` + ); + return result; + } + + const wksFolder = docManager.getWorkspaceFolder(doc); + if (!wksFolder) { + relatedFilesLogger.debug(logTarget, + 'neighborFiles.getNeighborFilesAndTraits', + `Failed to get the related files: ${uri} is not under the workspace folder` + ); + return result; + } + + const relatedFiles = await instantiationService.invokeFunction(getRelatedFilesAndTraits, + doc, + telemetryData, + cancellationToken, + data, + forceRelatedFilesComputation + ); + + if (relatedFiles.entries.size === 0) { + relatedFilesLogger.debug(logTarget, + 'neighborFiles.getNeighborFilesAndTraits', + `0 related files found for ${uri}` + ); + // make sure we include traits if there's any + result.traits.push(...relatedFiles.traits); + return result; + } + + relatedFiles.entries.forEach((uriToContentMap, type) => { + const addedDocs: SimilarFileInfo[] = []; + uriToContentMap.forEach((source, uri) => { + const relativePath = NeighborSource.getRelativePath(uri, wksFolder.uri); + if (!relativePath) { return; } + // Check that results.docs does not already contain an entry for the given uri. + if (result.docs.has(uri)) { return; } + const relatedFileDocInfo: SimilarFileInfo = { relativePath, uri, source }; + addedDocs.unshift(relatedFileDocInfo); + result.docs.set(uri, relatedFileDocInfo); + }); + + if (addedDocs.length > 0) { + result.neighborSource.set( + type, + addedDocs.map(doc => doc.uri.toString()) + ); + } + }); + result.traits.push(...relatedFiles.traits); + + return result; + } + + static basename(uri: string): string { + return decodeURIComponent(uri.replace(/[#?].*$/, '').replace(/^.*[/:]/, '')); + } + + /** + * Get the fileUri relative to the provided basePath + * or its basename if basePath is not its ancestor. + */ + static getRelativePath(fileUri: string, baseUri: string): string | undefined { + const parentURI = baseUri + .toString() + .replace(/[#?].*/, '') + .replace(/\/?$/, '/'); + if (fileUri.toString().startsWith(parentURI)) { + return fileUri.toString().slice(parentURI.length); + } + return NeighborSource.basename(fileUri); + } +} + +export function isIncludeNeighborFilesActive(accessor: ServicesAccessor, languageId: string, telemetryData: TelemetryWithExp): boolean { + const featuresService = accessor.get(ICompletionsFeaturesService); + return featuresService.includeNeighboringFiles(languageId, telemetryData); +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/openTabFiles.ts b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/openTabFiles.ts new file mode 100644 index 0000000..a9dc51f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/openTabFiles.ts @@ -0,0 +1,85 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { sortByAccessTimes } from '../../documentTracker'; +import { TextDocumentContents } from '../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { + INeighborSource, + NeighborSource, + NeighboringFileType, + NeighborsCollection, + considerNeighborFile, +} from './neighborFiles'; + +export class OpenTabFiles implements INeighborSource { + constructor(@ICompletionsTextDocumentManagerService readonly docManager: ICompletionsTextDocumentManagerService) { } + + private truncateDocs( + docs: readonly TextDocumentContents[], + uri: string, + languageId: string, + maxNumNeighborFiles: number + ): NeighborsCollection { + const openFiles: NeighborsCollection = new Map(); + let totalLen = 0; + for (const doc of docs) { + if (totalLen + doc.getText().length > NeighborSource.MAX_NEIGHBOR_AGGREGATE_LENGTH) { + continue; + } + + if ( + doc.uri.startsWith('file:') && + uri.startsWith('file:') && + doc.uri !== uri && + considerNeighborFile(languageId, doc.detectedLanguageId) + ) { + openFiles.set(doc.uri.toString(), { + uri: doc.uri.toString(), + relativePath: this.docManager.getRelativePath(doc), + source: doc.getText(), + }); + totalLen += doc.getText().length; + } + + if (openFiles.size >= maxNumNeighborFiles) { + break; + } + } + return openFiles; + } + + /** + * Get the neighbor files. Current it supports open editors. + * @param uri The uri of the current open file. + * @param languageId The language id of the current open file. + * @param maxNumNeighborFiles The max number of neighbor files to return. + * @returns Include 2 items. + * 1. The merged unique documents, which is not exceeding MAX_NEIGHBOR_FILES. + * 2. For each neighbor type, the files that are included in the merged unique documents. + */ + async getNeighborFiles( + uri: string, + languageId: string, + maxNumNeighborFiles: number + ): Promise<{ docs: NeighborsCollection; neighborSource: Map<NeighboringFileType, string[]> }> { + let neighborFiles: NeighborsCollection = new Map(); + const neighborSource = new Map<NeighboringFileType, string[]>(); + neighborFiles = this.truncateDocs( + sortByAccessTimes(await this.docManager.textDocuments()), + uri, + languageId, + maxNumNeighborFiles + ); + neighborSource.set( + NeighboringFileType.OpenTabs, + Array.from(neighborFiles.keys()).map(uri => uri.toString()) + ); + return { + docs: neighborFiles, + neighborSource: neighborSource, + }; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/relatedFiles.ts b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/relatedFiles.ts new file mode 100644 index 0000000..eb1caff --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/relatedFiles.ts @@ -0,0 +1,391 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IIgnoreService } from '../../../../../../../platform/ignore/common/ignoreService'; +import { createServiceIdentifier } from '../../../../../../../util/common/services'; +import { URI } from '../../../../../../../util/vs/base/common/uri'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CancellationToken as ICancellationToken } from '../../../../types/src'; +import { ICompletionsFileSystemService } from '../../fileSystem'; +import { LRUCacheMap } from '../../helpers/cache'; +import { ICompletionsLogTargetService, Logger } from '../../logger'; +import { telemetry, TelemetryWithExp } from '../../telemetry'; +import { shortCircuit } from '../../util/shortCircuit'; +import { NeighboringFileType } from './neighborFiles'; + +export interface RelatedFilesDocumentInfo { + readonly uri: string; + readonly clientLanguageId: string; + data?: unknown; +} + +interface RelatedFilesTextDocument { + readonly uri: string; + readonly clientLanguageId: string; + readonly detectedLanguageId: string; +} + +export type RelatedFilesResponseEntry = { + type: NeighboringFileType; + uris: string[]; +}; + +export type RelatedFileTrait = { + name: string; + value: string; + includeInPrompt?: boolean; + promptTextOverride?: string; +}; + +export type RelatedFilesResponse = { + entries: RelatedFilesResponseEntry[]; + traits?: RelatedFileTrait[]; +}; + +type RelatedFiles = { + entries: RelatedFilesType; + traits: RelatedFileTrait[]; +}; + +export type RelatedFilesType = Map<NeighboringFileType, Map<string, string>>; + +export const EmptyRelatedFilesResponse: RelatedFilesResponse = { entries: [], traits: [] }; + +const EmptyRelatedFiles: RelatedFiles = { + entries: new Map<NeighboringFileType, Map<string, string>>(), + traits: [], +}; + +type TimestampEntry = { timestamp: number; retryCount: number }; +// A map with an expiration time for each key. Keys are removed upon get() time. +// Note: the size() function is not being used, but if it does, be aware that it is +// counting expired keys. This ensures a constant time execution time. +export class PromiseExpirationCacheMap<T> extends LRUCacheMap<string, Promise<T>> { + // Hold the time an entry is cached the first time. The entries in this map are only removed + // upon a get() call when the eviction time elapsed. + _cacheTimestamps: Map<string, TimestampEntry> = new Map(); + + constructor( + size: number, + private readonly defaultEvictionTimeMs: number = 2 * 60 * 1000 // 2 minutes + ) { + super(size); + } + + bumpRetryCount(key: string): number { + const ts = this._cacheTimestamps.get(key); + if (ts) { + return ++ts.retryCount; + } else { + this._cacheTimestamps.set(key, { timestamp: Date.now(), retryCount: 0 }); + return 0; + } + } + + override has(key: string): boolean { + if (this.isValid(key)) { + return super.has(key); + } else { + this.deleteExpiredEntry(key); + return false; + } + } + + override get(key: string): Promise<T> | undefined { + const entry = super.get(key); + if (this.isValid(key)) { + return entry; + } else { + this.deleteExpiredEntry(key); + return undefined; + } + } + + override set(key: string, value: Promise<T>): this { + const ret = super.set(key, value); + if (!this.isValid(key)) { + this._cacheTimestamps.set(key, { timestamp: Date.now(), retryCount: 0 }); + } + return ret; + } + + override clear() { + super.clear(); + this._cacheTimestamps.clear(); + } + + // A cache entry is considered valid if its lifetime is less than the default cache eviction time. + private isValid(key: string): boolean { + const ts = this._cacheTimestamps.get(key); + return ts !== undefined && Date.now() - ts.timestamp < this.defaultEvictionTimeMs; + } + + private deleteExpiredEntry(key: string): void { + if (this._cacheTimestamps.has(key)) { + this._cacheTimestamps.delete(key); + } + super.delete(key); + } +} + +export const relatedFilesLogger = new Logger('relatedFiles'); +const lruCacheSize = 1000; + +class RelatedFilesProviderFailure extends Error { + constructor() { + super('The provider failed providing the list of relatedFiles'); + } +} + +export const ICompletionsRelatedFilesProviderService = createServiceIdentifier<ICompletionsRelatedFilesProviderService>('ICompletionsRelatedFilesProviderService'); +export interface ICompletionsRelatedFilesProviderService { + readonly _serviceBrand: undefined; + getRelatedFilesResponse( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined + ): Promise<RelatedFilesResponse | undefined>; + getRelatedFiles( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined + ): Promise<RelatedFiles | undefined>; +} + +/** + * Class for getting the related files to the current active file (implemented in the extension or the agent). + */ +export abstract class RelatedFilesProvider implements ICompletionsRelatedFilesProviderService { + declare _serviceBrand: undefined; + constructor( + @IInstantiationService protected readonly instantiationService: IInstantiationService, + @IIgnoreService protected readonly ignoreService: IIgnoreService, + @ICompletionsLogTargetService protected readonly logTarget: ICompletionsLogTargetService, + @ICompletionsFileSystemService protected readonly fileSystemService: ICompletionsFileSystemService, + ) { } + + // Returns the related files for the given document. + // An exception or `undefined` may be returned if a return value cannot be provided for some reason (e.g. failures). + abstract getRelatedFilesResponse( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined + ): Promise<RelatedFilesResponse | undefined>; + + async getRelatedFiles( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined + ): Promise<RelatedFiles | undefined> { + // Try/catch-ing around getRelatedFilesResponse is not useful: it is up to the + // concrete implementation of getRelatedFilesResponse to handle exceptions. If + // they are thrown at this point, let them pass through up to the memoize() to + // handle cache eviction. + const response = await this.getRelatedFilesResponse(docInfo, telemetryData, cancellationToken); + if (response === undefined) { return undefined; } + + const result: RelatedFiles = { + entries: new Map<NeighboringFileType, Map<string, string>>(), + traits: response.traits ?? [], + }; + + for (const entry of response.entries) { + let uriToContentMap = result.entries.get(entry.type); + if (!uriToContentMap) { + uriToContentMap = new Map<string, string>(); + result.entries.set(entry.type, uriToContentMap); + } + for (const uri of entry.uris) { + try { + relatedFilesLogger.debug(this.logTarget, `Processing ${uri}`); + + let content = await this.getFileContent(uri); + if (!content || content.length === 0) { + relatedFilesLogger.debug(this.logTarget, `Skip ${uri} due to empty content or loading issue.`); + continue; + } + + if (await this.isContentExcluded(uri, content)) { + relatedFilesLogger.debug(this.logTarget, `Skip ${uri} due content exclusion.`); + continue; + } + + content = RelatedFilesProvider.dropBOM(content); + uriToContentMap.set(uri, content); + } catch (e) { + relatedFilesLogger.warn(this.logTarget, e); + } + } + } + + return result; + } + + protected async getFileContent(uri: string): Promise<string | undefined> { + try { + return this.fileSystemService.readFileString(uri); + } catch (e) { + relatedFilesLogger.debug(this.logTarget, e); + } + + return undefined; + } + + private async isContentExcluded(uri: string, content: string): Promise<boolean> { + try { + return this.ignoreService.isCopilotIgnored(URI.parse(uri)); + } catch (e) { + this.instantiationService.invokeFunction(acc => relatedFilesLogger.exception(acc, e, 'isContentExcluded')); + } + + // Default to being excluded if encountered error + return true; + } + + private static dropBOM(content: string): string { + // Note: charCodeAt() converts the UTF8 BOM to UTF16 BOM (`0xefbbbf` to `0xfeff`), + // so only the latter must be checked. + if (content.charCodeAt(0) === 0xfeff) { + return content.slice(1); + } + + return content; + } +} + +const defaultMaxRetryCount: number = 3; // times the cache may be evicted and refreshed (e.g. a retry) +const lruCache: PromiseExpirationCacheMap<RelatedFiles> = new PromiseExpirationCacheMap(lruCacheSize); + +/** + * Given a document, gets a list of related files which are cached (memoized). + * If the result is not already cached, then the lookup is made based purely upon docInfo and then cached. + * */ +async function getRelatedFiles( + accessor: ServicesAccessor, + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined, + relatedFilesProvider: ICompletionsRelatedFilesProviderService +): Promise<RelatedFiles> { + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const startTime = performance.now(); + let result: RelatedFiles | undefined; + try { + result = await relatedFilesProvider.getRelatedFiles(docInfo, telemetryData, cancellationToken); + } catch (error) { + instantiationService.invokeFunction(acc => relatedFilesLogger.exception(acc, error, '.getRelatedFiles')); + result = undefined; + } + + if (result === undefined) { + const retryCount = lruCache.bumpRetryCount(docInfo.uri); + if (retryCount >= defaultMaxRetryCount) { + // Retry limit reached, cache and return an empty list. + result = EmptyRelatedFiles; + } else { + result = undefined; + } + } + + const elapsedTime = performance.now() - startTime; + relatedFilesLogger.debug(logTarget, + result !== undefined + ? `Fetched ${[...result.entries.values()] + .map(value => value.size) + .reduce((total, current) => total + current, 0)} related files for '${docInfo.uri + }' in ${elapsedTime}ms.` + : `Failing fetching files for '${docInfo.uri}' in ${elapsedTime}ms.` + ); + + // If the provider failed, throwing will let memoize() evict the key from the cache, and will be tried again. + if (result === undefined) { + throw new RelatedFilesProviderFailure(); + } + return result; +} + +let getRelatedFilesWithCacheAndTimeout = function ( + accessor: ServicesAccessor, + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp, + cancellationToken: ICancellationToken | undefined, + relatedFilesProvider: ICompletionsRelatedFilesProviderService +): Promise<RelatedFiles> { + const id = `${docInfo.uri}`; + if (lruCache.has(id)) { + return lruCache.get(id)!; + } + let result = getRelatedFiles(accessor, docInfo, telemetryData, cancellationToken, relatedFilesProvider); + if (result instanceof Promise) { + result = result.catch(error => { + lruCache.delete(id); + throw error; + }); + } + lruCache.set(id, result); + return result; +}; + +getRelatedFilesWithCacheAndTimeout = shortCircuit( + getRelatedFilesWithCacheAndTimeout, + 200, // max milliseconds + EmptyRelatedFiles +); + +/** + * For a given document, it provides a list of related files and traits + * @param ctx The context. + * @param doc The document information. + * @param telemetryData Object used to send telemetry and check experimentation options. + * @param cancellationToken The cancellation token. + * @param data Additional arbitrary data to be passed to the provider. + * @param forceComputation Set true to force computation by skipping cache and timeout. + * @returns Related files and traits. + */ +export async function getRelatedFilesAndTraits( + accessor: ServicesAccessor, + doc: RelatedFilesTextDocument, + telemetryData: TelemetryWithExp, + cancellationToken?: ICancellationToken, + data?: unknown, + forceComputation: boolean = false +): Promise<RelatedFiles> { + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const relatedFilesProvider = accessor.get(ICompletionsRelatedFilesProviderService); + + let relatedFiles = EmptyRelatedFiles; + try { + const docInfo: RelatedFilesDocumentInfo = { + uri: doc.uri, + clientLanguageId: doc.clientLanguageId, + data: data, + }; + relatedFiles = forceComputation + ? await instantiationService.invokeFunction(getRelatedFiles, docInfo, telemetryData, cancellationToken, relatedFilesProvider) + : await instantiationService.invokeFunction(getRelatedFilesWithCacheAndTimeout, + docInfo, + telemetryData, + cancellationToken, + relatedFilesProvider + ); + } catch (error) { + relatedFiles = EmptyRelatedFiles; + if (error instanceof RelatedFilesProviderFailure) { + instantiationService.invokeFunction(telemetry, 'getRelatedFilesList', telemetryData); + } + } + + relatedFilesLogger.debug(logTarget, + relatedFiles !== null && relatedFiles !== undefined + ? `Fetched following traits ${relatedFiles.traits + .map(trait => `{${trait.name} : ${trait.value}}`) + .join('')} for '${doc.uri}'` + : `Failing fecthing traits for '${doc.uri}'.` + ); + + return relatedFiles; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/test/neighborFiles.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/test/neighborFiles.test.ts new file mode 100644 index 0000000..6f20de0 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/test/neighborFiles.test.ts @@ -0,0 +1,300 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { IIgnoreService } from '../../../../../../../../platform/ignore/common/ignoreService'; +import { SyncDescriptor } from '../../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService } from '../../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { accessTimes } from '../../../documentTracker'; +import { ExpTreatmentVariables } from '../../../experiments/expConfig'; +import { ICompletionsFileSystemService } from '../../../fileSystem'; +import { ICompletionsLogTargetService } from '../../../logger'; +import { TelemetryWithExp } from '../../../telemetry'; +import { createLibTestingContext } from '../../../test/context'; +import { TestTextDocumentManager } from '../../../test/textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; +import { NeighboringFileType, NeighborSource } from '../neighborFiles'; +import { OpenTabFiles } from '../openTabFiles'; +import { + ICompletionsRelatedFilesProviderService, + RelatedFilesDocumentInfo, + RelatedFilesProvider, + RelatedFilesResponse, + RelatedFilesResponseEntry, + RelatedFileTrait, +} from '../relatedFiles'; + +const TIMEOUT = 1000; + +const WKS_ROOTFOLDER = 'file:///test'; + +const FILE_A = 'file:///test/a.py'; +const FILE_A_TEXT = '# file a'; + +const FILE_B = 'file:///test/b.py'; +const FILE_B_TEXT = '# file b'; + +const FILE_C = 'file:///test/c.py'; +const FILE_C_TEXT = '# file c'; + +const FILE_D = 'file:///test/d.py'; +const FILE_D_TEXT = '# file d'; + +const FILE_E = 'file:///test/test2/e.py'; +const FILE_E_TEXT = '# file e'; + +const FILE_F = 'file:///test/test2/f.py'; +const FILE_F_TEXT = '# file f'; + +const FILE_G = 'file:///test/test3/test4/g.py'; +const FILE_G_TEXT = '# file g'; + +const FILE_I = 'file:///test/test2/i.py'; +const FILE_I_TEXT = '# file i'; + +const FILE_J = 'file:///test/test2/j.js'; +const FILE_J_TEXT = '# file j'; + +const FILE_K = 'file:///test/test2/k.md'; +const FILE_K_TEXT = '# file k'; + +const FILE_R = 'file:///test/test2/r.jsx'; +const FILE_R_TEXT = '# file r'; + +const FILE_S = 'file:///test/test2/s.js'; +const FILE_S_TEXT = '# file s'; + +const FILE_T = 'file:///test/test2/t.js'; +const FILE_T_TEXT = '# file t'; + +const CURRENT_TIME_STAMP = Date.now(); +const CURSOR_HISTORY_FOR_TEST: { uri: string; offset: number; timestamp: number; text: string }[] = [ + { uri: FILE_C, offset: 0, timestamp: CURRENT_TIME_STAMP - 14, text: FILE_C_TEXT }, + { uri: FILE_C, offset: 0, timestamp: CURRENT_TIME_STAMP - 13, text: FILE_C_TEXT }, + { uri: FILE_C, offset: 0, timestamp: CURRENT_TIME_STAMP - 12, text: FILE_C_TEXT }, + { uri: FILE_A, offset: 0, timestamp: CURRENT_TIME_STAMP - 11, text: FILE_A_TEXT }, + { uri: FILE_D, offset: 0, timestamp: CURRENT_TIME_STAMP - 10, text: FILE_D_TEXT }, + { uri: FILE_D, offset: 0, timestamp: CURRENT_TIME_STAMP - 9, text: FILE_D_TEXT }, + { uri: FILE_D, offset: 0, timestamp: CURRENT_TIME_STAMP - 8, text: FILE_D_TEXT }, + { uri: FILE_D, offset: 0, timestamp: CURRENT_TIME_STAMP - 7, text: FILE_D_TEXT }, + { uri: FILE_A, offset: 0, timestamp: CURRENT_TIME_STAMP - 6, text: FILE_A_TEXT }, + { uri: FILE_C, offset: 0, timestamp: CURRENT_TIME_STAMP - 5, text: FILE_C_TEXT }, + { uri: FILE_B, offset: 0, timestamp: CURRENT_TIME_STAMP - 4, text: FILE_B_TEXT }, + { uri: FILE_B, offset: 0, timestamp: CURRENT_TIME_STAMP - 3, text: FILE_B_TEXT }, + { uri: FILE_B, offset: 0, timestamp: CURRENT_TIME_STAMP - 2, text: FILE_B_TEXT }, + { uri: FILE_J, offset: 0, timestamp: CURRENT_TIME_STAMP - 1, text: FILE_J_TEXT }, + { uri: FILE_A, offset: 0, timestamp: CURRENT_TIME_STAMP, text: FILE_A_TEXT }, +]; + +const OPEN_FILES_FOR_TEST: { uri: string; timestamp: number; text: string; language: string }[] = [ + { uri: FILE_T, timestamp: CURRENT_TIME_STAMP - 7, text: FILE_T_TEXT, language: 'javascript' }, + { uri: FILE_D, timestamp: CURRENT_TIME_STAMP - 6, text: FILE_D_TEXT, language: 'python' }, + { uri: FILE_R, timestamp: CURRENT_TIME_STAMP - 3, text: FILE_R_TEXT, language: 'javascriptreact' }, + { uri: FILE_C, timestamp: CURRENT_TIME_STAMP - 4, text: FILE_C_TEXT, language: 'python' }, + { uri: FILE_J, timestamp: CURRENT_TIME_STAMP - 3, text: FILE_J_TEXT, language: 'javascript' }, + { uri: FILE_K, timestamp: CURRENT_TIME_STAMP - 2, text: FILE_K_TEXT, language: 'markdown' }, + { uri: FILE_B, timestamp: CURRENT_TIME_STAMP - 1, text: FILE_B_TEXT, language: 'python' }, + { uri: FILE_A, timestamp: CURRENT_TIME_STAMP, text: FILE_A_TEXT, language: 'python' }, +]; + +const WORKSPACE_FILES_FOR_TEST: { uri: string; text: string; language: string }[] = [ + { uri: FILE_E, text: FILE_E_TEXT, language: 'python' }, + { uri: FILE_D, text: FILE_D_TEXT, language: 'python' }, + { uri: FILE_F, text: FILE_F_TEXT, language: 'python' }, + { uri: FILE_G, text: FILE_G_TEXT, language: 'python' }, + { uri: FILE_I, text: FILE_I_TEXT, language: 'python' }, + { uri: FILE_J, text: FILE_J_TEXT, language: 'javascript' }, + { uri: FILE_K, text: FILE_K_TEXT, language: 'markdown' }, + { uri: FILE_S, text: FILE_S_TEXT, language: 'javascript' }, + { uri: FILE_T, text: FILE_T_TEXT, language: 'javascript' }, +]; + +const CURRENT_FILE = CURSOR_HISTORY_FOR_TEST[CURSOR_HISTORY_FOR_TEST.length - 1].uri; + +const MAX_NUM_NEIGHBORING_FILES = 20; +const DEFAULT_FILE_LANGUAGE = 'python'; + +suite('neighbor files tests', function () { + this.timeout(TIMEOUT); + const accessor = createLibTestingContext().createTestingAccessor(); + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + + const workspaceTextDocumentManager = accessor.get(IInstantiationService).createInstance(TestTextDocumentManager); + for (const file of WORKSPACE_FILES_FOR_TEST) { + workspaceTextDocumentManager.setDiskContents(file.uri, file.text); + } + + workspaceTextDocumentManager.setTextDocument(FILE_I, DEFAULT_FILE_LANGUAGE, FILE_I_TEXT); + + for (const file of OPEN_FILES_FOR_TEST) { + tdm.setTextDocument(file.uri, file.language, file.text); + } + + setup(() => { + accessTimes.clear(); + for (const file of OPEN_FILES_FOR_TEST) { + accessTimes.set(file.uri, file.timestamp); + } + }); + + test('Test open files', async function () { + const at = accessTimes; + console.log('Access times:', at); + const ns = new OpenTabFiles(tdm); + const { docs, neighborSource } = await ns.getNeighborFiles( + CURRENT_FILE, + DEFAULT_FILE_LANGUAGE, + MAX_NUM_NEIGHBORING_FILES + ); + assert.strictEqual(docs.size, 3); + assert.strictEqual(docs.has(FILE_B), true); + assert.strictEqual(docs.has(FILE_C), true); + assert.strictEqual(docs.has(FILE_D), true); + assert.strictEqual(neighborSource.has(NeighboringFileType.CursorMostCount), false); + assert.strictEqual(neighborSource.has(NeighboringFileType.CursorMostRecent), false); + assert.strictEqual(neighborSource.has(NeighboringFileType.OpenTabs), true); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.length, 3); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.shift(), FILE_B); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.shift(), FILE_C); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.shift(), FILE_D); + }); + + test('Test open files file limit', async function () { + const ns = new OpenTabFiles(tdm); + const { docs } = await ns.getNeighborFiles(CURRENT_FILE, DEFAULT_FILE_LANGUAGE, /* maxNumNeighborFiles */ 1); + assert.strictEqual(docs.size, 1); + }); + + test('Include neighboring files for aliased languages', async function () { + const ns = new OpenTabFiles(tdm); + const { docs } = await ns.getNeighborFiles(CURRENT_FILE, 'javascript', MAX_NUM_NEIGHBORING_FILES); + + assert.ok(docs.has(FILE_J)); + assert.ok(docs.has(FILE_R)); + }); +}); + +suite('NeighborSource.getRelativePath tests', function () { + test('should return the relative path', function () { + const file = 'file:/path/to/file.txt'; + const base = 'file:/path/to'; + const relativePath = NeighborSource.getRelativePath(file, base); + assert.strictEqual(relativePath, 'file.txt'); + + const sshFile = 'ssh://path/to/file.txt'; + const sshBase = 'ssh:'; + const relativeSshPath = NeighborSource.getRelativePath(sshFile, sshBase); + assert.strictEqual(relativeSshPath, '/path/to/file.txt'); + }); + + test('should return the basename of the file if not related to the basePath (and should not add ".." to the path either)', function () { + { + const file = 'gopher:/path/to/file.txt'; + const base = 'https://path/to'; + const relativePath = NeighborSource.getRelativePath(file, base); + assert.strictEqual(relativePath, 'file.txt'); + } + + { + const file = 'file:/path/to/file.txt'; + const base = 'file://path/to/sibling'; + const relativePath = NeighborSource.getRelativePath(file, base); + assert.strictEqual(relativePath, 'file.txt'); + const relativePath2 = NeighborSource.getRelativePath(base, file); + assert.strictEqual(relativePath2, 'sibling'); + } + + { + const file = ''; + const base = 'file:///'; + const relativePath = NeighborSource.getRelativePath(file, base); + assert.strictEqual(relativePath, ''); + } + + { + const file = ''; + const base = ''; + const relativePath = NeighborSource.getRelativePath(file, base); + assert.strictEqual(relativePath, ''); + } + }); +}); + +suite('Neighbor files exclusion tests', function () { + class MockedRelatedFilesProvider extends RelatedFilesProvider { + constructor( + private readonly relatedFiles: RelatedFilesResponseEntry[], + private readonly traits: RelatedFileTrait[] = [{ name: 'testTraitName', value: 'testTraitValue' }], + @IInstantiationService instantiationService: IInstantiationService, + @IIgnoreService ignoreService: IIgnoreService, + @ICompletionsLogTargetService logTarget: ICompletionsLogTargetService, + @ICompletionsFileSystemService fileSystemService: ICompletionsFileSystemService, + ) { + super(instantiationService, ignoreService, logTarget, fileSystemService); + } + + async getRelatedFilesResponse( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp + ): Promise<RelatedFilesResponse | undefined> { + return Promise.resolve({ + entries: this.relatedFiles, + traits: this.traits, + }); + } + + override getFileContent(uri: string): Promise<string | undefined> { + // we are not asserting on file content, so just return a dummy text + return Promise.resolve('dummy text'); + } + } + + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsRelatedFilesProviderService, new SyncDescriptor(MockedRelatedFilesProvider, [[], [{ name: 'testTraitName', value: 'testTraitValue' }]])); + + const accessor = serviceCollection.createTestingAccessor(); + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: WKS_ROOTFOLDER }]); + + for (const file of OPEN_FILES_FOR_TEST) { + accessTimes.set(file.uri, file.timestamp); + } + + const workspaceTextDocumentManager = accessor.get(IInstantiationService).createInstance(TestTextDocumentManager); + for (const file of WORKSPACE_FILES_FOR_TEST) { + workspaceTextDocumentManager.setDiskContents(file.uri, file.text); + } + + workspaceTextDocumentManager.setTextDocument(FILE_I, DEFAULT_FILE_LANGUAGE, FILE_I_TEXT); + + for (const file of OPEN_FILES_FOR_TEST) { + tdm.setTextDocument(file.uri, file.language, file.text); + } + + test('Test with related files excluded', async function () { + NeighborSource.reset(); + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.ExcludeRelatedFiles] = true; + const { docs, neighborSource, traits } = await NeighborSource.getNeighborFilesAndTraits( + accessor, + FILE_J, + 'javascript', + telemetryWithExp, + undefined, + undefined, + true + ); + + assert.strictEqual(docs.size, 2); + assert.strictEqual(docs.has(FILE_T), true); + assert.strictEqual(docs.has(FILE_R), true); + assert.strictEqual(neighborSource.size, 1); + assert.strictEqual(neighborSource.has(NeighboringFileType.OpenTabs), true); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.length, 2); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.shift(), FILE_R); + assert.strictEqual(neighborSource.get(NeighboringFileType.OpenTabs)?.shift(), FILE_T); + assert.strictEqual(traits.length, 0); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/test/relatedFiles.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/test/relatedFiles.test.ts new file mode 100644 index 0000000..8b604fc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/similarFiles/test/relatedFiles.test.ts @@ -0,0 +1,666 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import Sinon from 'sinon'; +import type { CancellationToken } from 'vscode'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { SyncDescriptor } from '../../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { accessTimes } from '../../../documentTracker'; +import { ExpTreatmentVariables } from '../../../experiments/expConfig'; +import { TelemetryWithExp } from '../../../telemetry'; +import { createLibTestingContext } from '../../../test/context'; +import { TestTextDocumentManager } from '../../../test/textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../../textDocumentManager'; +import { getFsPath } from '../../../util/uri'; +import { CompositeRelatedFilesProvider, ProviderCallback } from '../compositeRelatedFilesProvider'; +import { NeighborSource, NeighboringFileType } from '../neighborFiles'; +import { + ICompletionsRelatedFilesProviderService, + PromiseExpirationCacheMap, + RelatedFilesDocumentInfo, RelatedFilesResponse, + RelatedFilesType, + getRelatedFilesAndTraits +} from '../relatedFiles'; + +suite('PromiseExpirationCacheMap', function () { + const r: RelatedFilesType = new Map<NeighboringFileType, Map<string, string>>(); + const x = Promise.resolve(r); + test('should add and retrieve entries using set and get methods', function () { + const cache = new PromiseExpirationCacheMap<RelatedFilesType>(2); + cache.set('a', x); + cache.set('b', x); + cache.set('c', x); + assert.equal(cache.get('b'), x); + assert.equal(cache.get('c'), x); + assert.equal(cache.get('a'), undefined, 'a should have been removed from the cache'); + assert.equal(cache.size, 2); + }); + + test('get() should evict expired cache entries', async function () { + const cache = new PromiseExpirationCacheMap<RelatedFilesType>(3, 10); + cache.set('a', x); + cache.set('b', x); + await new Promise(resolve => setTimeout(resolve, 20)); + cache.set('c', x); + // size does count existing expired entries. + assert.equal(cache.size, 3); + assert.equal(cache.get('a'), undefined); + assert.equal(cache.get('b'), undefined); + assert.equal(cache.get('c'), x); + assert.equal(cache.size, 1); + await new Promise(resolve => setTimeout(resolve, 20)); + assert.equal(cache.get('c'), undefined); + assert.equal(cache.size, 0); + }); + + test('has() should evict expired cache entries', async function () { + const cache = new PromiseExpirationCacheMap<RelatedFilesType>(7, 10); + cache.set('a', x); + cache.set('b', x); + await new Promise(resolve => setTimeout(resolve, 20)); + cache.set('c', x); + assert.equal(cache.has('c'), true); + assert.equal(cache.get('c'), x); + assert.equal(cache.has('a'), false); + assert.equal(cache.has('b'), false); + assert.equal(cache.get('a'), undefined); + assert.equal(cache.get('b'), undefined); + await new Promise(resolve => setTimeout(resolve, 20)); + assert.equal(cache.has('c'), false); + assert.equal(cache.get('c'), undefined); + }); + + test('clear works', function () { + const cache = new PromiseExpirationCacheMap<RelatedFilesType>(2); + cache.set('a', x); + cache.set('b', x); + cache.clear(); + assert.equal(cache.get('a'), undefined); + assert.equal(cache.get('b'), undefined); + assert.equal(cache.size, 0); + }); +}); +function createOpenFiles(root: string, timestamp: number) { + const FILE_D = `${root}/d.py`; + const FILE_D_TEXT = '# file d'; + + const FILE_E = `${root}/e.cs`; + const FILE_E_TEXT = '// file e'; + + const FILE_R = `${root}/relative/r.jsx`; + const FILE_R_TEXT = '// file r'; + + const FILE_J = `${root}/relative/j.js`; + const FILE_J_TEXT = '// file j'; + + const FILE_K = `${root}/relative/k.md`; + const FILE_K_TEXT = '# file k'; + + return [ + { uri: FILE_D, timestamp: timestamp - 6, text: FILE_D_TEXT, language: 'python' }, + { uri: FILE_E, timestamp: timestamp - 4, text: FILE_E_TEXT, language: 'csharp' }, + { uri: FILE_R, timestamp: timestamp - 3, text: FILE_R_TEXT, language: 'javascriptreact' }, + { uri: FILE_J, timestamp: timestamp - 3, text: FILE_J_TEXT, language: 'javascript' }, + { uri: FILE_K, timestamp: timestamp - 2, text: FILE_K_TEXT, language: 'markdown' }, + ]; +} + +suite('relatedFiles tests', function () { + const TIMEOUT = 1000; + const DEFAULT_FILE_LANGUAGE = 'cpp'; + const CURRENT_TIME_STAMP = Date.now(); + const WKS_ROOTFOLDER = 'file:///test'; + this.timeout(TIMEOUT); + const OPEN_FILES_FOR_TEST = createOpenFiles(WKS_ROOTFOLDER, CURRENT_TIME_STAMP); + + test('Test scenario where 4 files provided by the C++ related files provider are identical to 2 provided by the OpenTabs`s neighborSource', async function () { + function getHeaderFileContent(uri: string) { + return `// file ${getFsPath(uri)}`; + } + const CPP_NONOPENTAB_HEADERS: string[] = []; + const CPP_OPENTAB_HEADERS: string[] = []; + for (let i = 0; i < 2; i++) { CPP_OPENTAB_HEADERS.push(`${WKS_ROOTFOLDER}/relative/cppheader${i + 1}.h`); } + for (let i = 2; i < 4; i++) { CPP_NONOPENTAB_HEADERS.push(`${WKS_ROOTFOLDER}/relative/cppheader${i + 1}.h`); } + const CPP_ALL_HEADERS: string[] = CPP_NONOPENTAB_HEADERS.concat(CPP_OPENTAB_HEADERS); + const CURRENT_TIME_STAMP = Date.now(); + + const FILE_CPP = `${WKS_ROOTFOLDER}/relative/main.cpp`; + const FILE_CPP_TEXT = '// file main.cpp'; + OPEN_FILES_FOR_TEST.push({ + uri: FILE_CPP, + timestamp: CURRENT_TIME_STAMP, + text: FILE_CPP_TEXT, + language: 'cpp', + }); + + // Add the files provided by OpenTabs that are also provided by the C++ relatedFiles provider. + for (const openTabHeader of CPP_OPENTAB_HEADERS) { + OPEN_FILES_FOR_TEST.push({ + uri: openTabHeader, + timestamp: CURRENT_TIME_STAMP, + text: getHeaderFileContent(openTabHeader), + language: 'cpp', + }); + } + + const DEFAULT_FILE_LANGUAGE = 'cpp'; + + class MockedCppRelatedFilesProvider extends CompositeRelatedFilesProvider { + override getRelatedFilesResponse( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp + ): Promise<RelatedFilesResponse | undefined> { + const uris = CPP_ALL_HEADERS; + return Promise.resolve({ entries: [{ type: NeighboringFileType.RelatedCpp, uris }] }); + } + + override getFileContent(uri: string): Promise<string | undefined> { + return Promise.resolve(getHeaderFileContent(uri)); + } + } + + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsRelatedFilesProviderService, new SyncDescriptor(MockedCppRelatedFilesProvider)); + serviceCollection.define(ICompletionsTextDocumentManagerService, new SyncDescriptor(TestTextDocumentManager)); + const accessor = serviceCollection.createTestingAccessor(); + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + NeighborSource.reset(); + + // Mock up the workspace folders. + tdm.init([{ uri: WKS_ROOTFOLDER }]); + + accessTimes.clear(); + for (const file of OPEN_FILES_FOR_TEST) { + accessTimes.set(file.uri, file.timestamp); + tdm.setTextDocument(file.uri, file.language, file.text); + } + + const telemetry = TelemetryWithExp.createEmptyConfigForTesting(); + + const result = await NeighborSource.getNeighborFilesAndTraits(accessor, FILE_CPP, DEFAULT_FILE_LANGUAGE, telemetry); + + // 4 header files, two provided by the OpenTabs neightborSource, and two provided by the C++ relatedFiles provider. + assert.strictEqual(result.docs.size, 4); + for (const file of CPP_ALL_HEADERS) { + assert.strictEqual(result.docs.has(file), true); + } + assert.strictEqual(result.neighborSource.has(NeighboringFileType.RelatedCpp), true); + assert.strictEqual(result.neighborSource.has(NeighboringFileType.OpenTabs), true); + for (const file of CPP_OPENTAB_HEADERS) { + assert.strictEqual(result.neighborSource.get(NeighboringFileType.OpenTabs)?.includes(file), true); + assert.strictEqual(result.neighborSource.get(NeighboringFileType.RelatedCpp)?.includes(file), false); + } + for (const file of CPP_NONOPENTAB_HEADERS) { + assert.strictEqual(result.neighborSource.get(NeighboringFileType.RelatedCpp)?.includes(file), true); + assert.strictEqual(result.neighborSource.get(NeighboringFileType.OpenTabs)?.includes(file), false); + } + }); + + test('Test scenarios where the C++ related files provider fails', async function () { + const DUMMY_OPEN_CPPFILE = 'file:///test/relative/main2.cpp'; + const DUMMY_RELATED_FILE = 'file:///test/relative/related-file.cpp'; + const RETRY_COUNT = 3; + + enum FailureType { + WithException, + WithUndefined, + NoFailure, + } + + class MockedCppRelatedFilesProvider extends CompositeRelatedFilesProvider { + override getRelatedFilesResponse( + _docInfo: RelatedFilesDocumentInfo, + _telemetryData: TelemetryWithExp, + _cancellationToken: CancellationToken | undefined + ): Promise<RelatedFilesResponse | undefined> { + switch (this._failureType) { + case FailureType.WithException: + return Promise.reject(new Error('The provider failed to provide the related files')); + case FailureType.WithUndefined: + return Promise.resolve(undefined); + case FailureType.NoFailure: + return Promise.resolve({ + entries: [{ type: NeighboringFileType.RelatedCpp, uris: [DUMMY_RELATED_FILE] }], + }); + } + } + + override getFileContent(uri: string): Promise<string | undefined> { + return Promise.resolve('// C++ dummy content'); + } + + setFailWith(type: FailureType): void { + this._failureType = type; + } + + private _failureType: FailureType = FailureType.NoFailure; + } + + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsRelatedFilesProviderService, new SyncDescriptor(MockedCppRelatedFilesProvider)); + serviceCollection.define(ICompletionsTextDocumentManagerService, new SyncDescriptor(TestTextDocumentManager)); + const accessor = serviceCollection.createTestingAccessor(); + + const cppProvider = accessor.get(ICompletionsRelatedFilesProviderService) as MockedCppRelatedFilesProvider; + const telemetry = TelemetryWithExp.createEmptyConfigForTesting(); + const cppProviderGetMock = Sinon.spy(cppProvider, 'getRelatedFilesResponse'); + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.init([{ uri: WKS_ROOTFOLDER }]); + const DUMMY_CPP = 'file:///test/relative/dummy.cpp'; + accessTimes.set(DUMMY_CPP, CURRENT_TIME_STAMP); + tdm.setTextDocument(DUMMY_CPP, DEFAULT_FILE_LANGUAGE, DUMMY_RELATED_FILE); + + // One time init of NeighborSource singleton. + NeighborSource.reset(); + // An empty list is cached when the retryCount limit is reached for a given URI. + let result = undefined; + for (let i = 0; i < RETRY_COUNT; i++) { + cppProvider.setFailWith(RETRY_COUNT % 2 === 0 ? FailureType.WithException : FailureType.WithUndefined); + result = await NeighborSource.getNeighborFilesAndTraits(accessor, DUMMY_CPP, DEFAULT_FILE_LANGUAGE, telemetry); + assert.strictEqual(result.neighborSource.has(NeighboringFileType.RelatedCpp), false); + assert.strictEqual(cppProviderGetMock.callCount, 1); + assert.strictEqual(cppProviderGetMock.calledOnce, true); + cppProviderGetMock.resetHistory(); + } + cppProvider.setFailWith(FailureType.WithException); + for (let i = 0; i < RETRY_COUNT; i++) { + result = await NeighborSource.getNeighborFilesAndTraits(accessor, DUMMY_CPP, DEFAULT_FILE_LANGUAGE, telemetry); + assert.strictEqual(result.neighborSource.has(NeighboringFileType.RelatedCpp), false); + assert.strictEqual(cppProviderGetMock.calledOnce, false); + cppProviderGetMock.resetHistory(); + } + + // The actual result is cached when retrieval works within the given retryCount limit. + accessTimes.set(DUMMY_OPEN_CPPFILE, CURRENT_TIME_STAMP); + tdm.setTextDocument(DUMMY_OPEN_CPPFILE, DEFAULT_FILE_LANGUAGE, DUMMY_RELATED_FILE); + cppProvider.setFailWith(FailureType.WithException); + for (let i = 0; i < RETRY_COUNT - 1; i++) { + result = await NeighborSource.getNeighborFilesAndTraits( + accessor, + DUMMY_OPEN_CPPFILE, + DEFAULT_FILE_LANGUAGE, + telemetry + ); + assert.strictEqual(result.neighborSource.has(NeighboringFileType.RelatedCpp), false); + assert.strictEqual(cppProviderGetMock.calledOnce, true); + cppProviderGetMock.resetHistory(); + } + cppProvider.setFailWith(FailureType.NoFailure); + cppProviderGetMock.resetHistory(); + result = await NeighborSource.getNeighborFilesAndTraits( + accessor, + DUMMY_OPEN_CPPFILE, + DEFAULT_FILE_LANGUAGE, + telemetry + ); + assert.strictEqual(result.neighborSource.has(NeighboringFileType.RelatedCpp), true); + assert.strictEqual(result.docs.has(DUMMY_RELATED_FILE), true); + assert.strictEqual(cppProviderGetMock.calledOnce, true); + cppProviderGetMock.resetHistory(); + result = await NeighborSource.getNeighborFilesAndTraits( + accessor, + DUMMY_OPEN_CPPFILE, + DEFAULT_FILE_LANGUAGE, + telemetry + ); + assert.strictEqual(result.neighborSource.has(NeighboringFileType.RelatedCpp), true); + assert.strictEqual(result.docs.has(DUMMY_RELATED_FILE), true); + assert.strictEqual(cppProviderGetMock.calledOnce, false); + cppProviderGetMock.resetHistory(); + }); + + suite('CompositeRelatedFilesProvider', function () { + class TestCompositeRelatedFilesProvider extends CompositeRelatedFilesProvider { + override getFileContent(uri: string): Promise<string | undefined> { + if (uri.endsWith('.js') || uri.endsWith('.ts')) { + return Promise.resolve('// js dummy'); + } else if (uri.endsWith('.cs')) { + return Promise.resolve('// cs dummy'); + } + return Promise.resolve(undefined); + } + } + + async function compositeGetRelated( + providers: Array<{ + languageId: string; + extensionId: string; + callback: ProviderCallback; + }>, + telemetryWithExp: TelemetryWithExp, + filetype: 'csharp' | 'javascript' | 'python' = 'javascript', + cancel = false + ) { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsTextDocumentManagerService, new SyncDescriptor(TestTextDocumentManager)); + serviceCollection.define(ICompletionsRelatedFilesProviderService, new SyncDescriptor(TestCompositeRelatedFilesProvider)); + const accessor = serviceCollection.createTestingAccessor(); + + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + // Mock up the workspace folders. + tdm.init([{ uri: WKS_ROOTFOLDER }]); + const composite = accessor.get(ICompletionsRelatedFilesProviderService) as TestCompositeRelatedFilesProvider; + for (const { extensionId, languageId, callback } of providers) { + composite.registerRelatedFilesProvider(extensionId, languageId, callback); + } + const OPEN_FILES_FOR_TEST = createOpenFiles(WKS_ROOTFOLDER, Date.now()); + + const closedFiles = OPEN_FILES_FOR_TEST.map(f => ({ ...f, uri: f.uri.replace('.', '2.') })); + + for (const file of OPEN_FILES_FOR_TEST) { + accessTimes.set(file.uri, file.timestamp); + } + + for (const file of closedFiles) { + tdm.setDiskContents(file.uri, file.text); + } + + for (const file of OPEN_FILES_FOR_TEST) { + tdm.setTextDocument(file.uri, file.language, file.text); + } + + const uri = OPEN_FILES_FOR_TEST[filetype === 'javascript' ? 3 : 1].uri; + const doc = await tdm.getTextDocument({ uri }); + assert.ok(doc, `missing text document ${uri}`); + const wksFolder = tdm.getWorkspaceFolder(doc); + assert.ok(wksFolder, `missing workspace folder for ${uri}`); + + const cts = new CancellationTokenSource(); + if (cancel) { + cts.cancel(); + } + return (await getRelatedFilesAndTraits(accessor, doc, telemetryWithExp, cts.token, undefined, true)).entries; + } + + test('zero registered providers returns nothing', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated([], telemetryWithExp); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Typescript provider returns no files for JS file', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'typescript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '2.')] }, + ], + }), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Javascript provider returns nothing when cancelled', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url, context, token) => + Promise.resolve({ + entries: token.isCancellationRequested + ? [] + : [{ type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '2.')] }], + }), + }, + ], + telemetryWithExp, + 'javascript', + /*cancel*/ true + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Javascript provider returns a file for JS file', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '2.')] }, + ], + }), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual( + relatedFiles, + new Map([ + [NeighboringFileType.RelatedTypeScript, new Map([['file:///test/relative/j2.js', '// js dummy']])], + ]) + ); + }); + test('Javascript and C# providers only fire Typescript provider for JS', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'csharp', + extensionId: 'ms-dotnettools.csharp', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedCSharpRoslyn, uris: [url.replace('.', '2.')] }, + ], + }), + }, + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '3.')] }, + ], + }), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual( + relatedFiles, + new Map([ + [NeighboringFileType.RelatedTypeScript, new Map([['file:///test/relative/j3.js', '// js dummy']])], + ]) + ); + }); + test('multiple registration of Typescript providers for JS only returns one file', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '2.')] }, + ], + }), + }, + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '3.')] }, + ], + }), + }, + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '4.')] }, + ], + }), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual( + relatedFiles, + new Map([ + [NeighboringFileType.RelatedTypeScript, new Map([['file:///test/relative/j4.js', '// js dummy']])], + ]) + ); + }); + test('C# provider returns a file for .cs file', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'csharp', + extensionId: 'ms-dotnettools.csharp', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedCSharpRoslyn, uris: [url.replace('.', '2.')] }, + ], + }), + }, + ], + telemetryWithExp, + 'csharp' + ); + assert.deepStrictEqual( + relatedFiles, + new Map([[NeighboringFileType.RelatedCSharpRoslyn, new Map([['file:///test/e2.cs', '// cs dummy']])]]) + ); + }); + test('C# provider returns no files for JS file', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'csharp', + extensionId: 'ms-dotnettools.csharp', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedCSharpRoslyn, uris: [url.replace('.', '2.')] }, + ], + }), + }, + ], + telemetryWithExp, + 'javascript' + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Provider that throws returns no files', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = true; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => Promise.reject(new Error(`Error providing files for ${url}`)), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Inactive Typescript provider returns no related files', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = false; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '4.')] }, + ], + }), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Inactive Typescript provider returns no related files with general flag enabled', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = false; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = false; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCode] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'javascript', + extensionId: 'vscode.typescript-language-features', + callback: (url: string) => + Promise.resolve({ + entries: [ + { type: NeighboringFileType.RelatedTypeScript, uris: [url.replace('.', '4.')] }, + ], + }), + }, + ], + telemetryWithExp + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + test('Python provider returns related files with general flag enabled', async function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeTypeScript] = false; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCodeCSharp] = false; + telemetryWithExp.filtersAndExp.exp.variables[ExpTreatmentVariables.RelatedFilesVSCode] = true; + const relatedFiles = await compositeGetRelated( + [ + { + languageId: 'python', + extensionId: 'ms-python.python', + callback: (url: string) => + Promise.resolve({ + entries: [{ type: NeighboringFileType.RelatedOther, uris: [url.replace('.', '4.')] }], + }), + }, + ], + telemetryWithExp, + 'python' + ); + assert.deepStrictEqual(relatedFiles, new Map()); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistry.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistry.test.ts new file mode 100644 index 0000000..2090c46 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistry.test.ts @@ -0,0 +1,1582 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import Sinon from 'sinon'; +import { CancellationToken, CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { TestingServiceCollection } from '../../../../../../../platform/test/node/services'; +import { SyncDescriptor } from '../../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + ContextProvider, + ContextUsageStatistics, + DocumentContext, + ResolveRequest, + SupportedContextItem, + Trait, +} from '../../../../types/src/index'; +import { ConfigKey, ICompletionsConfigProvider, InMemoryConfigProvider } from '../../config'; +import { ICompletionsLogTargetService, LogLevel } from '../../logger'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { TestLogTarget } from '../../test/loggerHelpers'; +import { delay } from '../../util/async'; +import { ICompletionsRuntimeModeService, RuntimeMode } from '../../util/runtimeMode'; +import { ICompletionsContextProviderRegistryService, ResolvedContextItem } from '../contextProviderRegistry'; +import { TraitWithId } from '../contextProviders/contextItemSchemas'; +import { ContextProviderStatistics, ICompletionsContextProviderService } from '../contextProviderStatistics'; +import { TestContextProviderStatistics } from '../test/contextProviderStatistics'; +import { ICompletionsFeaturesService } from '../../experiments/featuresService'; + +suite('ContextProviderRegistry', function () { + let accessor: ServicesAccessor; + let serviceCollection: TestingServiceCollection; + let registry: ICompletionsContextProviderRegistryService; + let statistics: TestContextProviderStatistics; + let testLogTarget: TestLogTarget; + let telemetryData: TelemetryWithExp; + let clock: Sinon.SinonFakeTimers; + + const defaultDocumentContext: DocumentContext = { + uri: 'file:///test.txt', + languageId: 'md', + version: 1, + offset: 0, + position: { line: 0, character: 0 }, + }; + + const traitProvider: ContextProvider<Trait> = { + id: 'traitProvider', + selector: ['*'], + resolver: { + resolve: () => { + return Promise.resolve([ + { + name: 'trait1', + value: 'value1', + id: 'id1', + }, + ]); + }, + }, + }; + + setup(function () { + serviceCollection = createLibTestingContext(); + testLogTarget = new TestLogTarget(); + serviceCollection.define(ICompletionsLogTargetService, testLogTarget); + statistics = new TestContextProviderStatistics(); + serviceCollection.define(ICompletionsContextProviderService, new ContextProviderStatistics(() => statistics)); + accessor = serviceCollection.createTestingAccessor(); + + telemetryData = TelemetryWithExp.createEmptyConfigForTesting(); + registry = accessor.get(ICompletionsContextProviderRegistryService); + + // Enable all context providers for the suite. + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = '*'; + clock = Sinon.useFakeTimers(); + }); + + teardown(function () { + clock.restore(); + Sinon.restore(); + }); + + test('should register a context provider', function () { + registry.registerContextProvider(traitProvider); + + assert.deepStrictEqual(registry.providers.length, 1); + assert.deepStrictEqual(registry.providers[0].id, 'traitProvider'); + }); + + test('should not register a context provider with invalid name', function () { + const invalidProvider: ContextProvider<Trait> = { + id: 'in,validProvider', + selector: ['*'], + resolver: { + resolve: () => Promise.resolve([]), + }, + }; + assert.throws(() => registry.registerContextProvider(invalidProvider)); + }); + + test('should not register a duplicate context provider (by id)', function () { + registry.registerContextProvider(traitProvider); + assert.throws(() => registry.registerContextProvider(traitProvider)); + }); + + test('should unregister a context provider', function () { + registry.registerContextProvider(traitProvider); + assert.deepStrictEqual(registry.providers.length, 1); + + registry.unregisterContextProvider(traitProvider.id); + + assert.deepStrictEqual(registry.providers.length, 0); + }); + + test('resolving without providers should return an empty array', async function () { + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems, []); + }); + + test('negative matching providers should have none resolution', async function () { + const unmatchedProvider: ContextProvider<Trait> = { + id: 'unmatchedProvider', + selector: [{ language: 'typescript' }], + resolver: { + resolve: () => Promise.resolve([{ name: 'trait', value: 'value' }]), + }, + }; + + registry.registerContextProvider(unmatchedProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems, [ + { + providerId: 'unmatchedProvider', + matchScore: 0, + resolution: 'none', + resolutionTimeMs: 0, + data: [], + }, + ]); + }); + + for (const method of ['feature_flag', 'config']) { + for (const provider of ['enabledProvider', '*']) { + test(`enable ${provider} provider(s) via ${method}`, async function () { + if (method === 'feature_flag') { + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = provider; + } else { + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = ''; + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.ContextProviders, [provider]); + } + + const notEnabledProvider: ContextProvider<Trait> = { + id: 'notEnabledProvider', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'anothertrait', + value: 'anothervalue', + id: 'id1', + }, + ]), + }, + }; + const enabledProvider: ContextProvider<Trait> = { + id: 'enabledProvider', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'trait', + value: 'value', + id: 'id2', + }, + ]), + }, + }; + + registry.registerContextProvider(notEnabledProvider); + registry.registerContextProvider(enabledProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + if (provider === '*') { + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'notEnabledProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'anothertrait', value: 'anothervalue', id: 'id1', type: 'Trait' }], + }, + { + providerId: 'enabledProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'trait', value: 'value', id: 'id2', type: 'Trait' }], + }, + ]); + } else { + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'enabledProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'trait', value: 'value', id: 'id2', type: 'Trait' }], + }, + { + providerId: 'notEnabledProvider', + matchScore: 0, + resolution: 'none', + resolutionTimeMs: -1, + data: [], + }, + ]); + } + }); + } + } + + test('can resolve all providers', async function () { + const anotherTraitProvider: ContextProvider<Trait> = { + id: 'anotherTraitProvider', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'anotherTrait1', + value: 'anotherValue1', + id: 'id2', + }, + ]), + }, + }; + + registry.registerContextProvider(traitProvider); + registry.registerContextProvider(anotherTraitProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 2); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'traitProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'trait1', value: 'value1', id: 'id1', type: 'Trait' }], + }, + { + providerId: 'anotherTraitProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'anotherTrait1', value: 'anotherValue1', id: 'id2', type: 'Trait' }], + }, + ]); + }); + + test('providers that return no data are still considered resolved', async function () { + const noDataProvider: ContextProvider<Trait> = { + id: 'noDataProvider', + selector: ['*'], + resolver: { + resolve: () => Promise.resolve([]), + }, + }; + + registry.registerContextProvider(noDataProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems, [ + { + providerId: 'noDataProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: 0, + data: [], + }, + ]); + }); + + test('measures the resolution time', async function () { + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + resolve: async () => { + await clock.tickAsync(10); + return [{ name: 'trait1', value: 'value1' }]; + }, + }, + }; + + registry.registerContextProvider(slowProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.ok(resolvedContextItems[0].resolutionTimeMs >= 10); + }); + + test('should use passed IDs or assign one', async function () { + const traitProviderWithoutId: ContextProvider<Trait> = { + id: 'traitProviderWithoutId', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'traitWithoutId', + value: 'value', + }, + ]), + }, + }; + + registry.registerContextProvider(traitProvider); + registry.registerContextProvider(traitProviderWithoutId); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 2); + const [itemsWithId, itemsWithoutId] = resolvedContextItems; + + assert.ok(itemsWithoutId.data[0].id.length > 0); + assert.deepStrictEqual(itemsWithId.data[0].id, 'id1'); + }); + + test('context items with invalid IDs are replaced', async function () { + const traitProviderWithBadId: ContextProvider<Trait> = { + id: 'traitProviderWithBadId', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'traitWithBadId', + value: 'value', + id: 'in.valid', + }, + ]), + }, + }; + + registry.registerContextProvider(traitProviderWithBadId); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + const { data, resolutionTimeMs, ...rest } = resolvedContextItems[0]; + + assert.deepStrictEqual(rest, { + providerId: 'traitProviderWithBadId', + matchScore: 1, + resolution: 'full', + }); + assert.ok(resolutionTimeMs >= 0); + assert.deepStrictEqual(data.length, 1); + assert.ok(data[0].id.length > 0); + assert.notDeepStrictEqual(data[0].id, 'in.valid'); + }); + + test('context items with invalid importance are dropped', async function () { + const importances = [ + -1, // Out of range + 101, // Out of range + 99.9, // non-integer + 0.1, // non-integer + 50, // valid, + 0, // valid, + 100, // valid, + undefined, // valid + ]; + + const items: TraitWithId[] = []; + + for (const [ix, importance] of importances.entries()) { + items.push({ name: `trait${ix}`, value: `value${ix}`, importance, id: `${ix}`, type: 'Trait' }); + } + + const traitProviderWithBadId: ContextProvider<Trait> = { + id: 'traitProviderWithBadId', + selector: ['*'], + resolver: { + resolve: () => Promise.resolve(items), + }, + }; + + registry.registerContextProvider(traitProviderWithBadId); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + { + uri: 'file:///test.txt', + languageId: 'md', + version: 1, + offset: 0, + position: { line: 0, character: 0 }, + }, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + const { data } = resolvedContextItems[0]; + + assert.deepStrictEqual( + data.map(d => d.importance), + [50, 0, 100, undefined] + ); + }); + + test('context items with unsupported schema are dropped', async function () { + const traitProviderWithBadId: ContextProvider<Trait> = { + id: 'traitProviderWithBadId', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + 'hello' as unknown as TraitWithId, + { name: 'trait1', value: 'value1', id: '1', type: 'Trait' }, + { name: 'trait2', value: 'value2', id: '2', type: 'Trait' }, + ]), + }, + }; + + registry.registerContextProvider(traitProviderWithBadId); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + { + uri: 'file:///test.txt', + languageId: 'md', + version: 1, + offset: 0, + position: { line: 0, character: 0 }, + }, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + const { data } = resolvedContextItems[0]; + + assert.deepStrictEqual(data.length, 2); + }); + + test('context items with duplicate IDs are replaced', async function () { + const traitProviderWithDupeId: ContextProvider<Trait> = { + id: 'traitProviderWithDupeId', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'traitWithBadId1', + value: 'value', + id: 'id1', + }, + { + name: 'traitWithBadId2', + value: 'value', + id: 'id1', + }, + ]), + }, + }; + + registry.registerContextProvider(traitProviderWithDupeId); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + const { data, resolutionTimeMs, ...rest } = resolvedContextItems[0]; + + assert.deepStrictEqual(rest, { + providerId: 'traitProviderWithDupeId', + matchScore: 1, + resolution: 'full', + }); + assert.ok(resolutionTimeMs >= 0); + assert.deepStrictEqual(data.length, 2); + assert.deepStrictEqual(data[0].id, 'id1'); + assert.notDeepStrictEqual(data[1].id, 'id1'); + assert.ok(data[1].id.length > 0); + }); + + test('all providers are enabled in debug mode', async function () { + const serviceCollectionClone = serviceCollection.clone(); + + // Feature flag doesn't matter in debug mode + telemetryData.filtersAndExp.exp.variables.copilotcontextproviders = ''; + serviceCollectionClone.define(ICompletionsRuntimeModeService, RuntimeMode.fromEnvironment(false, [], { GITHUB_COPILOT_DEBUG: 'true' })); + const accessor = serviceCollectionClone.createTestingAccessor(); + const registry = accessor.get(ICompletionsContextProviderRegistryService); + + const anotherTraitProvider: ContextProvider<Trait> = { + id: 'anotherTraitProvider', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'anotherTrait1', + value: 'anotherValue1', + id: '1234', + }, + ]), + }, + }; + + registry.registerContextProvider(traitProvider); + registry.registerContextProvider(anotherTraitProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 2); + }); + + test('does not resolve providers if already cancelled', async function () { + registry.registerContextProvider(traitProvider); + const cts = new CancellationTokenSource(); + cts.cancel(); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData, + cts.token + ); + + assert.deepStrictEqual(resolvedContextItems.length, 0); + }); + + test('supports non-array providers', async function () { + const flatTraitProvider: ContextProvider<Trait> = { + id: 'flatTraitProvider', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve({ + name: 'flatTrait1', + value: 'flatValue1', + id: 'id', + }), + }, + }; + + registry.registerContextProvider(flatTraitProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'flatTraitProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'flatTrait1', value: 'flatValue1', id: 'id', type: 'Trait' }], + }, + ]); + }); + + test('provider rejects', async function () { + testLogTarget = new TestLogTarget(); + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define(ICompletionsLogTargetService, testLogTarget); + const accessor = serviceCollectionClone.createTestingAccessor(); + const registry = accessor.get(ICompletionsContextProviderRegistryService); + + const errorProvider: ContextProvider<SupportedContextItem> = { + id: 'errorProvider', + selector: ['*'], + resolver: { + resolve: (_, token): Promise<never> => { + return Promise.reject(new Error('Intentional error')); + }, + }, + }; + + registry.registerContextProvider(errorProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'errorProvider', + matchScore: 1, + resolution: 'error', + resolutionTimeMs: -1, + data: [], + }, + ]); + // Logs the error + testLogTarget.assertHasMessageMatching(LogLevel.ERROR, /Error resolving context/); + }); + + test('provider cancels', async function () { + testLogTarget = new TestLogTarget(); + const serviceCollectionClone = serviceCollection.clone(); + serviceCollectionClone.define(ICompletionsLogTargetService, testLogTarget); + const accessor = serviceCollectionClone.createTestingAccessor(); + const registry = accessor.get(ICompletionsContextProviderRegistryService); + + const errorProvider: ContextProvider<SupportedContextItem> = { + id: 'errorProvider', + selector: ['*'], + resolver: { + resolve: (_, token): Promise<never> => { + return Promise.reject(new CancellationError()); + }, + }, + }; + + registry.registerContextProvider(errorProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'errorProvider', + matchScore: 1, + resolution: 'error', + resolutionTimeMs: -1, + data: [], + }, + ]); + // In this case, no error is expected + assert.ok(testLogTarget.isEmpty()); + }); + + test('asynciterable provider rejects', async function () { + const errorProvider: ContextProvider<Trait> = { + id: 'errorAsyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve(_, token) { + // Return something, which will be ignored + yield { name: 'trait', value: 'value' }; + // Return promise that rejects + return Promise.reject(new Error('Intentional error')); + }, + }, + }; + + registry.registerContextProvider(errorProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'errorAsyncIterableProvider', + matchScore: 1, + resolution: 'error', + resolutionTimeMs: -1, + data: [], + }, + ]); + // Logs the error + testLogTarget.assertHasMessageMatching(LogLevel.ERROR, /Error resolving context/); + }); + + test('asynciterable provider cancels', async function () { + const errorProvider: ContextProvider<Trait> = { + id: 'errorAsyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve(_, token) { + // Return something, which will be ignored + yield { name: 'trait', value: 'value' }; + return Promise.reject(new CancellationError()); + }, + }, + }; + + registry.registerContextProvider(errorProvider); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'errorAsyncIterableProvider', + matchScore: 1, + resolution: 'error', + resolutionTimeMs: -1, + data: [], + }, + ]); + // In this case, no error is expected + assert.ok(testLogTarget.isEmpty()); + }); + + test('sets resolution status of providers', async function () { + registry.registerContextProvider(traitProvider); + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.deepStrictEqual(statistics.lastResolution.get('traitProvider'), 'full'); + }); + + test('times out when a (promise-based) provider takes too long', async function () { + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + resolve: async () => { + await clock.tickAsync(1000); + return [{ name: 'trait1', value: 'value1' }]; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'slowProvider', + matchScore: 1, + resolution: 'none', + resolutionTimeMs: -1, + data: [], + }, + ]); + assert.deepStrictEqual(statistics.lastResolution.get('slowProvider'), 'none'); + }); + + test('timeout is passed correctly', async function () { + clock.tick(1000); + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.ContextProviderTimeBudget, 100); + + let providerRequest: ResolveRequest | undefined; + const logOnlyProvider: ContextProvider<Trait> = { + id: 'logOnlyProvider', + selector: ['*'], + resolver: { + resolve: r => { + providerRequest = r; + return Promise.resolve([]); + }, + }, + }; + registry.registerContextProvider(logOnlyProvider); + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(providerRequest); + assert.deepStrictEqual(providerRequest.timeoutEnd, 1100); + assert.deepEqual(providerRequest.timeBudget, 100); + }); + + test('infinite timeout is passed correctly', async function () { + clock.tick(1000); + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.ContextProviderTimeBudget, 0); + + let providerRequest: ResolveRequest | undefined; + const logOnlyProvider: ContextProvider<Trait> = { + id: 'logOnlyProvider', + selector: ['*'], + resolver: { + resolve: r => { + providerRequest = r; + return Promise.resolve([]); + }, + }, + }; + registry.registerContextProvider(logOnlyProvider); + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(providerRequest); + assert.deepStrictEqual(providerRequest.timeoutEnd, Number.MAX_SAFE_INTEGER); + assert.deepEqual(providerRequest.timeBudget, 0); + }); + + test('does not timeout when time budget set to 0', async function () { + const serviceCollectionClone = serviceCollection.clone(); + + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + resolve: () => Promise.resolve([{ name: 'trait1', value: 'value1', id: 'id' }]), + }, + }; + + serviceCollectionClone.define(ICompletionsRuntimeModeService, RuntimeMode.fromEnvironment(false, [], { GITHUB_COPILOT_DEBUG: 'true' })); + const accessor = serviceCollectionClone.createTestingAccessor(); + + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.ContextProviderTimeBudget, 0); + const registry = accessor.get(ICompletionsContextProviderRegistryService); + + registry.registerContextProvider(slowProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'slowProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [{ name: 'trait1', value: 'value1', id: 'id', type: 'Trait' }], + }, + ]); + assert.deepStrictEqual(statistics.lastResolution.get('slowProvider'), 'full'); + }); + + test('timeout cancels request to the provider (default)', async function () { + let interceptedCancellation: CancellationToken; + let interceptedRequest: ResolveRequest | undefined; + + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + resolve: async (request, token) => { + interceptedCancellation = token; + interceptedRequest = request; + await clock.tickAsync(1000); + return [{ name: 'trait1', value: 'value1' }]; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(interceptedCancellation!); + assert.ok(interceptedCancellation!.isCancellationRequested); + assert.deepStrictEqual(interceptedRequest?.timeBudget, 150); + }); + + test('timeout can be specified via EXP', async function () { + let interceptedCancellation: CancellationToken; + let interceptedRequest: ResolveRequest | undefined; + + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.contextProviderTimeBudget = () => 10; + + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + async *resolve(request, token) { + interceptedCancellation = token; + interceptedRequest = request; + await clock.tickAsync(5); + yield { name: 'asynctrait1', value: 'value1', id: 'id1' }; + await clock.tickAsync(4); + yield { name: 'asynctrait2', value: 'value2', id: 'id2' }; + await clock.tickAsync(5); + yield { name: 'asynctrait3', value: 'value3', id: 'id3' }; + }, + }, + }; + registry.registerContextProvider(slowProvider); + const result = await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(interceptedCancellation!); + assert.ok(interceptedCancellation!.isCancellationRequested); + assert.ok(result.length === 1); + assert.deepStrictEqual(result[0].data.length, 2); + assert.deepStrictEqual(interceptedRequest?.timeBudget, 10); + }); + + test('config timeout is preferred to EXP', async function () { + let interceptedCancellation: CancellationToken; + let interceptedRequest: ResolveRequest | undefined; + const featuresService = accessor.get(ICompletionsFeaturesService); + featuresService.contextProviderTimeBudget = () => 10; + const configProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + configProvider.setConfig(ConfigKey.ContextProviderTimeBudget, 20); + + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + async *resolve(request, token) { + interceptedCancellation = token; + interceptedRequest = request; + await clock.tickAsync(10); + yield { name: 'asynctrait1', value: 'value1', id: 'id1' }; + await clock.tickAsync(9); + yield { name: 'asynctrait2', value: 'value2', id: 'id2' }; + await clock.tickAsync(10); + yield { name: 'asynctrait3', value: 'value3', id: 'id3' }; + }, + }, + }; + registry.registerContextProvider(slowProvider); + const result = await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(interceptedCancellation!); + assert.ok(interceptedCancellation!.isCancellationRequested); + assert.ok(result.length === 1); + assert.deepStrictEqual(result[0].data.length, 2); + assert.deepStrictEqual(interceptedRequest?.timeBudget, 20); + }); + + test('(matching) providers run concurrently', async function () { + const firstProvider: ContextProvider<Trait> = { + id: 'firstProvider', + selector: ['*'], + resolver: { + async *resolve() { + await delay(140); + yield { name: 'trait1', value: 'value1', id: 'id1' }; + yield { name: 'trait2', value: 'value2', id: 'id2' }; + }, + }, + }; + + // Items from this provider will be processed first. + const secondProvider: ContextProvider<Trait> = { + id: 'secondProvider', + selector: [{ language: 'md' }], + resolver: { + async *resolve() { + await delay(20); + yield { name: 'trait3', value: 'value3', id: 'id3' }; // Will make it + await delay(120); + yield { name: 'trait4', value: 'value4', id: 'id4' }; // Will make it + await delay(20); + yield { name: 'trait5', value: 'value5', id: 'id5' }; // Will not make it + }, + }, + }; + + // This provider will be ignored because it doesn't match + const thirdProvider: ContextProvider<Trait> = { + id: 'thirdProvider', + selector: [{ language: 'typescript' }], + resolver: { + async *resolve() { + await delay(75); + yield { name: 'trait6', value: 'value6', id: 'id6' }; + }, + }, + }; + + registry.registerContextProvider(firstProvider); + registry.registerContextProvider(secondProvider); + registry.registerContextProvider(thirdProvider); + + const resolvedContextItemsPromise = registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + await clock.runAllAsync(); + const resolvedContextItems = await resolvedContextItemsPromise; + + assert.deepStrictEqual(resolvedContextItems.length, 3); + assert.deepStrictEqual( + resolvedContextItems.map(c => c.providerId), + ['secondProvider', 'firstProvider', 'thirdProvider'] + ); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'secondProvider', + matchScore: 10, + resolution: 'partial', + resolutionTimeMs: -1, + data: [ + { name: 'trait3', value: 'value3', id: 'id3', type: 'Trait' }, + { name: 'trait4', value: 'value4', id: 'id4', type: 'Trait' }, + ], + }, + { + providerId: 'firstProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [ + { name: 'trait1', value: 'value1', id: 'id1', type: 'Trait' }, + { name: 'trait2', value: 'value2', id: 'id2', type: 'Trait' }, + ], + }, + { + providerId: 'thirdProvider', + matchScore: 0, + resolution: 'none', + resolutionTimeMs: -1, + data: [], + }, + ]); + + assert.deepStrictEqual(statistics.lastResolution.get('firstProvider'), 'full'); + assert.deepStrictEqual(statistics.lastResolution.get('secondProvider'), 'partial'); + }); + + test('supports asynciterable resolvers', async function () { + const asyncIterableProvider: ContextProvider<Trait> = { + id: 'asyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve() { + yield Promise.resolve({ name: 'asynctrait1', value: 'value1', id: 'id1' }); + yield Promise.resolve({ name: 'asynctrait2', value: 'value2', id: 'id2' }); + yield Promise.resolve({ name: 'asynctrait3', value: 'value3', id: 'id3' }); + }, + }, + }; + registry.registerContextProvider(asyncIterableProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'asyncIterableProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [ + { name: 'asynctrait1', value: 'value1', id: 'id1', type: 'Trait' }, + { name: 'asynctrait2', value: 'value2', id: 'id2', type: 'Trait' }, + { name: 'asynctrait3', value: 'value3', id: 'id3', type: 'Trait' }, + ], + }, + ]); + }); + + test('fallback context items are included if iterable timeout is hit', async function () { + let called = false; + + const asyncIterableProvider: ContextProvider<Trait> = { + id: 'asyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve() { + yield Promise.resolve({ name: 'asynctrait1', value: 'value1', id: 'id1' }); + yield Promise.resolve({ name: 'asynctrait2', value: 'value2', id: 'id2' }); + await clock.tickAsync(1000); // Timeout + yield Promise.resolve({ name: 'asynctrait3', value: 'value3', id: 'id3' }); + }, + resolveOnTimeout() { + called = true; + return [{ name: 'fallbacktrait', value: 'fallbackvalue', id: 'id4' }]; + }, + }, + }; + registry.registerContextProvider(asyncIterableProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.ok(called); + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'asyncIterableProvider', + matchScore: 1, + resolution: 'partial', + resolutionTimeMs: -1, + data: [ + { name: 'asynctrait1', value: 'value1', id: 'id1', type: 'Trait' }, + { name: 'asynctrait2', value: 'value2', id: 'id2', type: 'Trait' }, + { name: 'fallbacktrait', value: 'fallbackvalue', id: 'id4', type: 'Trait' }, + ], + }, + ]); + }); + + test('fallback context items are included if promise timeout is hit', async function () { + let called = false; + + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + async resolve() { + await clock.tickAsync(1000); // Timeout + return { name: 'trait', value: 'value', id: 'id1' }; + }, + resolveOnTimeout() { + called = true; + return [{ name: 'fallbacktrait', value: 'fallbackvalue', id: 'id2' }]; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.ok(called); + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'slowProvider', + matchScore: 1, + resolution: 'partial', + resolutionTimeMs: -1, + data: [{ name: 'fallbacktrait', value: 'fallbackvalue', id: 'id2', type: 'Trait' }], + }, + ]); + }); + + test('resolution remains none if no fallback items are provided', async function () { + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + async resolve() { + await clock.tickAsync(1000); // Timeout + return { name: 'trait', value: 'value', id: 'id1' }; + }, + resolveOnTimeout() { + return undefined; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'slowProvider', + matchScore: 1, + resolution: 'none', + resolutionTimeMs: -1, + data: [], + }, + ]); + }); + + test('fallback context items are not included if no timeout is hit', async function () { + let called = false; + + const asyncIterableProvider: ContextProvider<Trait> = { + id: 'asyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve() { + yield Promise.resolve({ name: 'asynctrait1', value: 'value1', id: 'id1' }); + yield Promise.resolve({ name: 'asynctrait2', value: 'value2', id: 'id2' }); + }, + resolveOnTimeout() { + called = true; + return [{ name: 'fallbacktrait', value: 'fallbackvalue', id: 'id4' }]; + }, + }, + }; + registry.registerContextProvider(asyncIterableProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.ok(!called); + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'asyncIterableProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [ + { name: 'asynctrait1', value: 'value1', id: 'id1', type: 'Trait' }, + { name: 'asynctrait2', value: 'value2', id: 'id2', type: 'Trait' }, + ], + }, + ]); + }); + + test('fallback context items are not included if no timeout is hit and no results', async function () { + let called = false; + + const asyncIterableProvider: ContextProvider<Trait> = { + id: 'asyncIterableProvider', + selector: ['*'], + resolver: { + resolve: () => Promise.resolve([]), + resolveOnTimeout() { + called = true; + return [{ name: 'fallbacktrait', value: 'fallbackvalue', id: 'id4' }]; + }, + }, + }; + registry.registerContextProvider(asyncIterableProvider); + + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + assert.ok(!called); + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'asyncIterableProvider', + matchScore: 1, + resolution: 'full', + resolutionTimeMs: -1, + data: [], + }, + ]); + }); + + test('times out when the first element of an (asynciterable-based) provider takes too long', async function () { + const slowProvider: ContextProvider<Trait> = { + id: 'slowAsyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve() { + await clock.tickAsync(1000); + yield { name: 'asynctrait1', value: 'value1' }; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + const startTime = Date.now(); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + // Allowing for a small error, even though we're using fake timers + assert.ok(Date.now() - startTime < 151); + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'slowAsyncIterableProvider', + matchScore: 1, + resolution: 'none', + resolutionTimeMs: -1, + data: [], + }, + ]); + assert.deepStrictEqual(statistics.lastResolution.get('slowAsyncIterableProvider'), 'none'); + }); + + test('times out when an (asynciterable-based) provider takes too long', async function () { + const slowProvider: ContextProvider<Trait> = { + id: 'slowAsyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve() { + yield { name: 'asynctrait1', value: 'value1', id: 'id1' }; + yield { name: 'asynctrait2', value: 'value2', id: 'id2' }; + await clock.tickAsync(1000); + yield { name: 'asynctrait3', value: 'value3', id: 'id3' }; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + const startTime = Date.now(); + const resolvedContextItems = await registry.resolveAllProviders( + '1234', + 'opId', + defaultDocumentContext, + telemetryData + ); + + // Allowing for a small error, even though we're using fake timers + assert.ok(Date.now() - startTime < 151); + assert.deepStrictEqual(resolvedContextItems.length, 1); + assert.deepStrictEqual(removeResolutionTime(resolvedContextItems), [ + { + providerId: 'slowAsyncIterableProvider', + matchScore: 1, + resolution: 'partial', + resolutionTimeMs: -1, + data: [ + { name: 'asynctrait1', value: 'value1', id: 'id1', type: 'Trait' }, + { name: 'asynctrait2', value: 'value2', id: 'id2', type: 'Trait' }, + ], + }, + ]); + testLogTarget.assertHasMessageMatching( + LogLevel.INFO, + /Context provider slowAsyncIterableProvider exceeded time budget/ + ); + assert.deepStrictEqual(statistics.lastResolution.get('slowAsyncIterableProvider'), 'partial'); + }); + + test('timeout cancels request to the (asynciterable-based) provider', async function () { + let interceptedCancellation: CancellationToken; + + const slowProvider: ContextProvider<Trait> = { + id: 'slowAsyncIterableProvider', + selector: ['*'], + resolver: { + async *resolve(_, token) { + interceptedCancellation = token; + yield { name: 'asynctrait1', value: 'value1' }; + yield { name: 'asynctrait2', value: 'value2' }; + await clock.tickAsync(1000); + yield { name: 'asynctrait3', value: 'value3' }; + }, + }, + }; + registry.registerContextProvider(slowProvider); + + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(interceptedCancellation!); + assert.ok(interceptedCancellation!.isCancellationRequested); + }); + + test('cancels requests when completion token is cancelled', async function () { + const cts = new CancellationTokenSource(); + let interceptedCancellation: CancellationToken; + + let providerEndTime = Date.now() - 1000; + let resolverEndTime = Date.now() + 1000; + const slowProvider: ContextProvider<Trait> = { + id: 'slowProvider', + selector: ['*'], + resolver: { + resolve: async (_, token): Promise<Trait[]> => { + interceptedCancellation = token; + await delay(15); + providerEndTime = Date.now(); + return Promise.resolve([{ name: 'trait1', value: 'value1' }]); + }, + }, + }; + registry.registerContextProvider(slowProvider); + + // record the time that resolution finishes + void registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData, cts.token).then(() => { + resolverEndTime = Date.now(); + }); + // trigger the cancellation token after 5ms + void delay(2).then(() => { + cts.cancel(); + }); + await clock.runAllAsync(); + + // the provider should have received the cancellation request + assert.ok(interceptedCancellation!); + assert.ok(interceptedCancellation.isCancellationRequested); + + // regardless of the provider's behavior, we end promptly when we receive the cancellation token + // In particular, resolution finishes before the provider + assert.ok(resolverEndTime < providerEndTime); + }); + + test('adds completion id to request', async function () { + registry.registerContextProvider(traitProvider); + const resolverSpy = Sinon.spy(traitProvider.resolver, 'resolve'); + + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(resolverSpy.calledOnce); + assert.deepStrictEqual(resolverSpy.lastCall.args[0].completionId, '1234'); + }); + + test('passes data when resolving', async function () { + const data = { foo: 'bar' }; + + registry.registerContextProvider(traitProvider); + const resolverSpy = Sinon.spy(traitProvider.resolver, 'resolve'); + + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData, undefined, data); + + assert.ok(resolverSpy.calledOnce); + assert.deepStrictEqual(resolverSpy.lastCall.args[0].data, data); + }); + + test('does not add statistics to the context on the first resolution', async function () { + registry.registerContextProvider(traitProvider); + const resolverSpy = Sinon.spy(traitProvider.resolver, 'resolve'); + + await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(resolverSpy.calledOnce); + assert.deepStrictEqual(resolverSpy.lastCall.args[0].previousUsageStatistics, undefined); + }); + + test('augments provider context with statistics from last round', async function () { + const serviceCollectionClone = serviceCollection.clone(); + const resolverSpy = Sinon.spy(traitProvider.resolver, 'resolve'); + serviceCollectionClone.define(ICompletionsContextProviderService, new SyncDescriptor(ContextProviderStatistics, [() => new TestContextProviderStatistics()])); + const accessor = serviceCollectionClone.createTestingAccessor(); + + const registry = accessor.get(ICompletionsContextProviderRegistryService); + registry.registerContextProvider(traitProvider); + + const statistics = accessor.get(ICompletionsContextProviderService); + const previousStatistics: ContextUsageStatistics = { usage: 'partial', resolution: 'full' }; + (statistics.getStatisticsForCompletion('previous_id') as TestContextProviderStatistics).statistics.set( + traitProvider.id, + previousStatistics + ); + await registry.resolveAllProviders('previous_id', 'opId', defaultDocumentContext, telemetryData); + (statistics.getStatisticsForCompletion('current_id') as TestContextProviderStatistics).statistics.set( + traitProvider.id, + { usage: 'none', resolution: 'none' } + ); + await registry.resolveAllProviders('current_id', 'opId', defaultDocumentContext, telemetryData); + + assert.deepStrictEqual(resolverSpy.firstCall.args[0].previousUsageStatistics, undefined); + assert.deepStrictEqual(resolverSpy.lastCall.args[0].previousUsageStatistics, previousStatistics); + }); + + test('caches results', async function () { + const resolverSpy = Sinon.spy(traitProvider.resolver, 'resolve'); + + const anotherTraitProvider: ContextProvider<Trait> = { + id: 'anotherTraitProvider', + selector: ['*'], + resolver: { + resolve: () => + Promise.resolve([ + { + name: 'anotherTrait1', + value: 'anotherValue1', + }, + ]), + }, + }; + + const anotherResolverSpy = Sinon.spy(anotherTraitProvider.resolver, 'resolve'); + + registry.registerContextProvider(traitProvider); + + const firstCall = await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(resolverSpy.calledOnce); + assert.ok(anotherResolverSpy.notCalled); + assert.deepStrictEqual(firstCall.length, 1); + + // Register another provider between calls to ensure more items are added. + registry.registerContextProvider(anotherTraitProvider); + + const secondCall = await registry.resolveAllProviders('1234', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(resolverSpy.calledOnce); + assert.ok(anotherResolverSpy.notCalled); + assert.deepStrictEqual(secondCall.length, 1); + assert.deepStrictEqual(firstCall, secondCall); + + const thirdCall = await registry.resolveAllProviders('5678', 'opId', defaultDocumentContext, telemetryData); + + assert.ok(resolverSpy.calledTwice); + assert.ok(anotherResolverSpy.calledOnce); + assert.deepStrictEqual(thirdCall.length, 2); + }); +}); + +// Utility function to test context items without worrying about non-deterministic fields +function removeResolutionTime(resolvedContextItems: ResolvedContextItem[]) { + return resolvedContextItems.map(i => { + i.resolutionTimeMs = -1; + return i; + }); +} + +class CancellationError extends Error { + constructor() { + super('Canceled'); + this.name = this.message; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts new file mode 100644 index 0000000..5199e0d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts @@ -0,0 +1,90 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { + getMultiLanguageContextProviderParamsFromActiveExperiments, + multiLanguageContextProviderParamsDefault, +} from '../contextProviderRegistryMultiLanguage'; + +suite('contextProviderRegistryMultiLanguage', function () { + let activeExperiments: Map<string, string | number | boolean | string[]>; + + setup(function () { + activeExperiments = new Map(); + }); + + suite('getMultiLanguageContextProviderConfigFromActiveExperiments', function () { + test('returns default config when no experiments are set', function () { + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(new Map()); + + assert.deepStrictEqual(result, multiLanguageContextProviderParamsDefault); + }); + + test('overrides defaults with experiment values', function () { + activeExperiments.set('mlcpMaxContextItems', '50'); + activeExperiments.set('mlcpMaxSymbolMatches', 30); + activeExperiments.set('mlcpEnableImports', true); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 50); + assert.strictEqual(result.mlcpMaxSymbolMatches, 30); + assert.strictEqual(result.mlcpEnableImports, true); + }); + + test('converts string values to appropriate types', function () { + activeExperiments.set('mlcpMaxContextItems', '25'); + activeExperiments.set('mlcpEnableImports', 'true'); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 25); + assert.strictEqual(result.mlcpEnableImports, true); + }); + + test('converts string values for false to appropriate types', function () { + activeExperiments.set('mlcpMaxContextItems', '25'); + activeExperiments.set('mlcpEnableImports', 'false'); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 25); + assert.strictEqual(result.mlcpEnableImports, false); + }); + + test('handles partial overrides', function () { + activeExperiments.set('mlcpEnableImports', true); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual( + result.mlcpMaxContextItems, + multiLanguageContextProviderParamsDefault.mlcpMaxContextItems + ); + assert.strictEqual( + result.mlcpMaxSymbolMatches, + multiLanguageContextProviderParamsDefault.mlcpMaxSymbolMatches + ); + assert.strictEqual(result.mlcpEnableImports, true); + }); + + test('converts falsy values correctly', function () { + activeExperiments.set('mlcpMaxContextItems', 0); + activeExperiments.set('mlcpEnableImports', false); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 0); + assert.strictEqual(result.mlcpEnableImports, false); + }); + + test('returns false for imports when not set', function () { + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpEnableImports, false); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts new file mode 100644 index 0000000..2bd19c2 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { ActiveExperiments } from '../contextProviderRegistry'; +import { fillInTsActiveExperiments, TS_CONTEXT_PROVIDER_ID } from '../contextProviderRegistryTs'; + +suite('contextProviderRegistryTs', function () { + let accessor: ServicesAccessor; + let activeExperiments: ActiveExperiments; + let telemetryData: TelemetryWithExp; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + activeExperiments = new Map(); + telemetryData = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryData.filtersAndExp.exp.variables['copilottscontextproviderparams'] = JSON.stringify({ + booleanProperty: true, + }); + }); + + test('does not add active experiments if no provider is active', function () { + fillInTsActiveExperiments(accessor, [], activeExperiments, telemetryData); + + assert.ok(activeExperiments.size === 0); + }); + + test('adds active experiments if TS provider is active', function () { + fillInTsActiveExperiments(accessor, [TS_CONTEXT_PROVIDER_ID], activeExperiments, telemetryData); + + assert.ok(activeExperiments.has('booleanProperty')); + assert.strictEqual(activeExperiments.get('booleanProperty'), true); + }); + + test('adds active experiments in debug mode', function () { + fillInTsActiveExperiments(accessor, ['*'], activeExperiments, telemetryData); + + assert.ok(activeExperiments.has('booleanProperty')); + assert.strictEqual(activeExperiments.get('booleanProperty'), true); + }); + + test('bad JSON is ignored', function () { + telemetryData.filtersAndExp.exp.variables['copilottscontextproviderparams'] = '{"badJSON": true'; + + fillInTsActiveExperiments(accessor, [TS_CONTEXT_PROVIDER_ID], activeExperiments, telemetryData); + + assert.ok(activeExperiments.size === 0); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts new file mode 100644 index 0000000..1e69c6a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts @@ -0,0 +1,256 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ResolutionStatus } from '../../../../types/src/index'; +import { TraitWithId } from '../contextProviders/contextItemSchemas'; +import { PromptMatcher } from '../contextProviderStatistics'; +import { TestContextProviderStatistics } from './contextProviderStatistics'; + +suite('contextProviderStatistics', function () { + let statistics: TestContextProviderStatistics; + const resolutions: ResolutionStatus[] = ['partial', 'full']; + + setup(function () { + statistics = new TestContextProviderStatistics(); + }); + + const trait1: TraitWithId = { + name: 'trait1', + value: 'value1', + id: '1234', + type: 'Trait', + }; + const trait2: TraitWithId = { + name: 'trait2', + value: 'value2', + id: '5678', + type: 'Trait', + }; + + test('can set expectations', function () { + statistics.addExpectations('bar', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + assert.deepStrictEqual(statistics.expectations.size, 1); + assert.deepStrictEqual(statistics.expectations.get('bar')?.length, 2); + }); + + test('can add expectations', function () { + statistics.addExpectations('bar', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + + const trait3: TraitWithId = { + name: 'trait3', + value: 'value3', + id: '9012', + type: 'Trait', + }; + const trait4: TraitWithId = { + name: 'trait4', + value: 'value4', + id: '3456', + type: 'Trait', + }; + + statistics.addExpectations('bar', [ + [trait3, 'included'], + [trait4, 'content_excluded'], + ]); + assert.deepStrictEqual(statistics.expectations.size, 1); + assert.deepStrictEqual(statistics.expectations.get('bar')?.length, 4); + }); + + test('computing match unsets expectations and resolution', function () { + statistics.addExpectations('bar', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('bar', 'full'); + assert.deepStrictEqual(statistics.expectations.size, 1); + assert.deepStrictEqual(statistics.lastResolution.size, 1); + + statistics.computeMatch([]); + + assert.deepStrictEqual(statistics.expectations.size, 0); + assert.deepStrictEqual(statistics.lastResolution.size, 0); + }); + + test('does not compute match for empty expectations', function () { + statistics.addExpectations('bar', []); + + statistics.computeMatch([]); + + assert.deepStrictEqual(statistics.statistics.size, 0); + }); + + for (const resolution of resolutions) { + test(`can match full expectations, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'included'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatcher: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 7, + source: trait1, + }, + { + expectedTokens: 10, + actualTokens: 10, + source: trait2, + }, + ]; + + statistics.computeMatch(promptMatcher); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'full'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }, + { id: '5678', usage: 'full', expectedTokens: 10, actualTokens: 10, type: 'Trait' }, + ]); + }); + + test(`can match partial expectations, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'included'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatchers: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 7, + source: trait1, + }, + { + expectedTokens: 10, + actualTokens: 5, + source: trait2, + }, + ]; + + statistics.computeMatch(promptMatchers); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'partial'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }, + { id: '5678', usage: 'partial', expectedTokens: 10, actualTokens: 5, type: 'Trait' }, + ]); + }); + + test(`full elision is no usage, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'included'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatchers: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 0, + source: trait1, + }, + { + expectedTokens: 10, + actualTokens: 0, + source: trait2, + }, + ]; + + statistics.computeMatch(promptMatchers); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'none'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'none', expectedTokens: 7, actualTokens: 0, type: 'Trait' }, + { id: '5678', usage: 'none', expectedTokens: 10, actualTokens: 0, type: 'Trait' }, + ]); + }); + + test(`some content excluded items make it partial, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatchers: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 7, + source: trait1, + }, + ]; + + statistics.computeMatch(promptMatchers); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'partial'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }, + { id: '5678', usage: 'none_content_excluded', type: 'Trait' }, + ]); + }); + + test(`all content excluded items make it none, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'content_excluded'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('foo', resolution); + + statistics.computeMatch([]); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'none'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'none_content_excluded', type: 'Trait' }, + { id: '5678', usage: 'none_content_excluded', type: 'Trait' }, + ]); + }); + } + + test('none resolution is always no match', function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait1, 'included'], + ]); + statistics.setLastResolution('foo', 'none'); + + statistics.computeMatch([]); + + const stats = statistics.get('foo'); + assert.deepStrictEqual(stats!, { usage: 'none', resolution: 'none' }); + }); + + test('error resolution is always no match', function () { + statistics.addExpectations('foo', [ + [trait1, 'content_excluded'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('foo', 'error'); + + statistics.computeMatch([]); + + const stats = statistics.get('foo'); + assert.deepStrictEqual(stats!, { usage: 'none', resolution: 'error' }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderStatistics.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderStatistics.ts new file mode 100644 index 0000000..683d213 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderStatistics.ts @@ -0,0 +1,24 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PerCompletionContextProviderStatistics } from '../contextProviderStatistics'; + +export class TestContextProviderStatistics extends PerCompletionContextProviderStatistics { + constructor() { + super(); + } + + get expectations() { + return this._expectations; + } + + get statistics() { + return this._statistics; + } + + get lastResolution() { + return this._lastResolution; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderTelemetry.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderTelemetry.ts new file mode 100644 index 0000000..cca3d04 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/contextProviderTelemetry.ts @@ -0,0 +1,26 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ContextProviderTelemetry } from '../contextProviderRegistry'; +import assert from 'assert'; + +export function assertContextProviderTelemetry( + actualContextProviderTelemetryJson: string, + expectedContextProviderTelemetry: Omit<ContextProviderTelemetry, 'resolutionTimeMs'>[] +) { + const parsedContextProviderTelemetry = JSON.parse(actualContextProviderTelemetryJson) as ContextProviderTelemetry[]; + // Assert that timing information is present + parsedContextProviderTelemetry.map(t => { + assert.ok(t.resolutionTimeMs >= 0); + }); + // Assert the rest of the telemetry (without timing) matches + assert.deepStrictEqual( + parsedContextProviderTelemetry.map(t => { + const { resolutionTimeMs, ...rest } = t; + return rest; + }), + expectedContextProviderTelemetry + ); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/determineTimeComplexity.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/determineTimeComplexity.ts new file mode 100644 index 0000000..0888aec --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/determineTimeComplexity.ts @@ -0,0 +1,78 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type Complexity = 'O(1)' | 'O(log n)' | 'O(sqrt(n))' | 'O(n)' | 'O(n log n)' | 'O(n^2)' | 'O(n^3)'; + +export interface ComplexityData { + n: number; + time: number; +} + +export interface ComplexityModel { + name: Complexity; + type: 'sublinear' | 'linear' | 'superlinear'; + basis: (n: number) => number; +} + +export interface ComplexityResult { + model: ComplexityModel; + coefficient: number; +} + +// Candidate basis functions +const models: ComplexityModel[] = [ + { name: 'O(1)', type: 'sublinear', basis: _n => 1 }, + { name: 'O(log n)', type: 'sublinear', basis: n => Math.log(n) }, + { name: 'O(n)', type: 'linear', basis: n => n }, + { name: 'O(n log n)', type: 'linear', basis: n => n * Math.log(n) }, + { name: 'O(n^2)', type: 'superlinear', basis: n => n * n }, + { name: 'O(n^3)', type: 'superlinear', basis: n => n * n * n }, + { name: 'O(sqrt(n))', type: 'sublinear', basis: n => Math.sqrt(n) }, +]; + +const constantComplexity = models.find(m => m.name === 'O(1)')!; + +export function determineTimeComplexity(data: ComplexityData[]): ComplexityResult { + if (data.length < 2) { + return { + model: constantComplexity, + coefficient: 0, + }; + } + + let bestModel: ComplexityModel = constantComplexity; + let bestError = Infinity; + let bestC = 0; + + for (const model of models) { + // Find best-fit coefficient C = sum(t_i ร‚ยท f_i) / sum(f_i^2) + let num = 0; + let den = 0; + for (const { n, time } of data) { + const f = model.basis(n); + num += time * f; + den += f * f; + } + const C = den > 0 ? num / den : 0; + + // Compute sum of squared errors + let sse = 0; + for (const { n, time } of data) { + const pred = C * model.basis(n); + sse += (time - pred) ** 2; + } + + if (sse < bestError) { + bestError = sse; + bestModel = model; + bestC = C; + } + } + + return { + model: bestModel, + coefficient: bestC, + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/parseBlock.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/parseBlock.test.ts new file mode 100644 index 0000000..4a1fcce --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/parseBlock.test.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { contextIndentationFromText } from '../parseBlock'; + +suite('Indentation', function () { + test('single line -> only current', function () { + assert.deepStrictEqual(contextIndentationFromText('x', 0, 'language'), { + prev: undefined, + current: 0, + next: undefined, + }); + }); + test('single line with line after -> only current & next', function () { + assert.deepStrictEqual(contextIndentationFromText('x\ny', 0, 'language'), { + prev: undefined, + current: 0, + next: 0, + }); + }); + test('after indent -> only current & prev', function () { + assert.deepStrictEqual(contextIndentationFromText('x\n y', 4, 'language'), { + prev: 0, + current: 1, + next: undefined, + }); + }); + test('after indent but before text -> only current from line above', function () { + assert.deepStrictEqual(contextIndentationFromText('x\n y', 3, 'language'), { + prev: undefined, + current: 0, + next: undefined, + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/prompt.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/prompt.test.ts new file mode 100644 index 0000000..af35c98 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/prompt.test.ts @@ -0,0 +1,296 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import dedent from 'ts-dedent'; +import { IIgnoreService } from '../../../../../../../platform/ignore/common/ignoreService'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + DEFAULT_MAX_COMPLETION_LENGTH, + DEFAULT_MAX_PROMPT_LENGTH, + DEFAULT_NUM_SNIPPETS, + DEFAULT_PROMPT_ALLOCATION_PERCENT, + DEFAULT_SUFFIX_MATCH_THRESHOLD, + PromptOptions, +} from '../../../../prompt/src/prompt'; +import { defaultSimilarFilesOptions } from '../../../../prompt/src/snippetInclusion/similarFiles'; +import { ExpTreatmentVariables } from '../../experiments/expConfig'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { MockIgnoreService } from '../../test/testContentExclusion'; +import { createTextDocument, InMemoryNotebookDocument, TestTextDocumentManager } from '../../test/textDocument'; +import { INotebookCell, IPosition } from '../../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../../textDocumentManager'; +import { CompletionsPromptRenderer } from '../components/completionsPromptRenderer'; +import { _copilotContentExclusion, _promptError, getPromptOptions } from '../prompt'; +import { extractPromptInternal } from './prompt'; + +suite('Prompt unit tests', function () { + let accessor: ServicesAccessor; + let sandbox: sinon.SinonSandbox; + + setup(function () { + sandbox = sinon.createSandbox(); + const serviceCollection = createLibTestingContext(); + serviceCollection.define(IIgnoreService, new MockIgnoreService()); + accessor = serviceCollection.createTestingAccessor(); + }); + + teardown(function () { + sandbox.restore(); + }); + + test('defaults to 8K max prompt length', async function () { + const content = 'function add()\n'; + const sourceDoc = createTextDocument('file:///foo.js', 'javascript', 0, content); + const cursorPosition: IPosition = { + line: 0, + character: 13, + }; + + const rendererStub = sandbox.stub(CompletionsPromptRenderer.prototype, 'render').throws('unspecified error'); + + const prompt = await extractPromptInternal( + accessor, + 'COMPLETION_ID', + sourceDoc, + cursorPosition, + TelemetryWithExp.createEmptyConfigForTesting() + ); + + assert.deepStrictEqual(prompt, _promptError); + assert.ok(rendererStub.calledOnce, 'should call renderer'); + assert.strictEqual( + rendererStub.firstCall.args[1].promptTokenLimit, + 8192 - DEFAULT_MAX_COMPLETION_LENGTH, + 'should default to 8192 max total tokens, 7692 max prompt tokens' + ); + }); + + test('default EXP prompt options are the same as default PromptOptions object', function () { + const promptOptionsFromExp = getPromptOptions(accessor, TelemetryWithExp.createEmptyConfigForTesting(), ''); + const defaultPromptOptions: PromptOptions = { + maxPromptLength: DEFAULT_MAX_PROMPT_LENGTH, + numberOfSnippets: DEFAULT_NUM_SNIPPETS, + similarFilesOptions: defaultSimilarFilesOptions, + suffixMatchThreshold: DEFAULT_SUFFIX_MATCH_THRESHOLD, + suffixPercent: DEFAULT_PROMPT_ALLOCATION_PERCENT.suffix, + }; + + assert.deepStrictEqual(promptOptionsFromExp, defaultPromptOptions); + }); + + test('default C++ EXP prompt options use tuned values', function () { + const promptOptionsFromExp: PromptOptions = getPromptOptions( + accessor, + TelemetryWithExp.createEmptyConfigForTesting(), + 'cpp' + ); + + assert.deepStrictEqual(promptOptionsFromExp.similarFilesOptions, { + snippetLength: 60, + threshold: 0.0, + maxTopSnippets: 16, + maxCharPerFile: 100000, + maxNumberOfFiles: 200, + maxSnippetsPerFile: 4, + useSubsetMatching: false, + }); + assert.deepStrictEqual(promptOptionsFromExp.numberOfSnippets, 16); + }); + + test('default Java EXP prompt options are correct', function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + const expVars = telemetryWithExp.filtersAndExp.exp.variables; + + Object.assign(expVars, { + [ExpTreatmentVariables.UseSubsetMatching]: true, + }); + + const promptOptionsFromExp = getPromptOptions(accessor, telemetryWithExp, 'java'); + assert.deepStrictEqual(promptOptionsFromExp.similarFilesOptions, { + snippetLength: 60, + threshold: 0.0, + maxTopSnippets: 4, + maxCharPerFile: 10000, + maxNumberOfFiles: 20, + maxSnippetsPerFile: 1, + useSubsetMatching: true, + }); + assert.deepStrictEqual(promptOptionsFromExp.numberOfSnippets, 4); + }); + + test('should return without a prompt if the file blocked by repository control', async function () { + (accessor.get(IIgnoreService) as MockIgnoreService).setAlwaysIgnore(); + + const content = 'function add()\n'; + const sourceDoc = createTextDocument('file:///foo.js', 'javascript', 0, content); + const cursorPosition: IPosition = { + line: 0, + character: 13, + }; + const response = await extractPromptInternal( + accessor, + 'COMPLETION_ID', + sourceDoc, + cursorPosition, + TelemetryWithExp.createEmptyConfigForTesting() + ); + assert.ok(response); + assert.strictEqual(response, _copilotContentExclusion); + }); + + test('prompt for ipython notebooks, using only the current cell language as shebang', async function () { + await assertPromptForCell( + accessor, + cells[4], + dedent( + `import math + +def add(a, b): + return a + b + +def product(c, d):` + ), + ['#!/usr/bin/env python3'] + ); + }); + + test('prompt for ipython notebooks, using only the current cell language for known language', async function () { + await assertPromptForCell( + accessor, + cells[5], + dedent( + `def product(c, d):` + ), + ['Language: julia'] + ); + }); + + test('prompt for ipython notebooks, using only the current cell language for unknown language', async function () { + await assertPromptForCell(accessor, cells[6], dedent(`foo bar baz`), ['Language: unknown-great-language']); + }); + + test('exception telemetry', async function () { + this.skip(); + /* todo@dbaeumer need to understand how we handle exception in chat + class TestExceptionTextDocumentManager extends TestTextDocumentManager { + override textDocuments() { + return Promise.reject(new Error('test error')); + } + } + const tdm = accessor.get(IInstantiationService).createInstance(TestExceptionTextDocumentManager); + tdm.setTextDocument('file:///a/1.py', 'python', 'import torch'); + ctx.forceSet(TextDocumentManager, tdm); + NeighborSource.reset(); + + const { reporter, enhancedReporter } = await withInMemoryTelemetry(ctx, async ctx => { + const document = createTextDocument('file:///a/2.py', 'python', 0, 'import torch'); + await extractPromptInternal( + ctx, + 'COMPLETION_ID', + document, + { line: 0, character: 0 }, + TelemetryWithExp.createEmptyConfigForTesting() + ); + }); + + assert.ok(reporter.hasException); + assert.deepStrictEqual( + reporter.firstException?.properties?.origin, + 'PromptComponents.CompletionsPromptFactory' + ); + assert.strictEqual(reporter.exceptions.length, 1); + + assert.ok(enhancedReporter.hasException); + assert.deepStrictEqual( + enhancedReporter.firstException?.properties?.origin, + 'PromptComponents.CompletionsPromptFactory' + ); + assert.strictEqual(enhancedReporter.exceptions.length, 1); + */ + }); +}); + +async function assertPromptForCell(accessor: ServicesAccessor, sourceCell: INotebookCell, expectedPrefix: string, expectedContext?: string[]) { + const notebook = new InMemoryNotebookDocument(cells); + const sourceDoc = sourceCell.document; + + (accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager).setNotebookDocument(sourceDoc, notebook); + + const cursorPosition: IPosition = { + line: 0, + character: sourceDoc.getText().length, + }; + const response = await extractPromptInternal( + accessor, + 'COMPLETION_ID', + sourceDoc, + cursorPosition, + TelemetryWithExp.createEmptyConfigForTesting() + ); + assert.ok(response); + assert.strictEqual(response.type, 'prompt'); + assert.strictEqual(response.prompt.prefix, expectedPrefix); + if (expectedContext !== undefined) { + assert.deepEqual(response.prompt.context, expectedContext); + } +} + +const cells: INotebookCell[] = [ + { + index: 1, + document: createTextDocument('file:///test/a.ipynb#1', 'python', 1, 'import math'), + metadata: {}, + kind: 2, + }, + { + index: 2, + document: createTextDocument( + 'file:///test/a.ipynb#2', + 'markdown', + 1, + 'This is an addition function\nIt is used to add two numbers' + ), + metadata: {}, + kind: 1, + }, + { + index: 3, + document: createTextDocument('file:///test/a.ipynb#3', 'python', 2, 'def add(a, b):\n return a + b'), + metadata: {}, + kind: 2, + }, + { + index: 4, + document: createTextDocument( + 'file:///test/a.ipynb#4', + 'markdown', + 2, + 'This is a product function\nYou guessed it: it multiplies two numbers' + ), + metadata: {}, + kind: 2, + }, + { + index: 5, + document: createTextDocument('file:///test/a.ipynb#5', 'python', 3, 'def product(c, d):'), + metadata: {}, + kind: 2, + }, + { + index: 6, + document: createTextDocument('file:///test/a.ipynb#6', 'julia', 3, 'def product(c, d):'), + metadata: {}, + kind: 2, + }, + { + index: 7, + document: createTextDocument('file:///test/a.ipynb#7', 'unknown-great-language', 3, 'foo bar baz'), + metadata: {}, + kind: 2, + }, +]; diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/prompt.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/prompt.ts new file mode 100644 index 0000000..e4694ac --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/prompt.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken } from 'vscode-languageserver-protocol'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { createCompletionState } from '../../completionState'; +import { getGhostText } from '../../ghostText/ghostText'; +import { TelemetryWithExp } from '../../telemetry'; +import { IPosition, ITextDocument } from '../../textDocument'; +import { ICompletionsContextProviderBridgeService } from '../components/contextProviderBridge'; +import { extractPrompt, ExtractPromptOptions } from '../prompt'; + +export async function extractPromptInternal( + accessor: ServicesAccessor, + completionId: string, + textDocument: ITextDocument, + position: IPosition, + telemetryWithExp: TelemetryWithExp, + promptOpts: ExtractPromptOptions = {} +) { + const completionState = createCompletionState(textDocument, position); + const contextProviderBridge = accessor.get(ICompletionsContextProviderBridgeService); + contextProviderBridge.schedule(completionState, completionId, 'opId', telemetryWithExp); + return extractPrompt(accessor, completionId, completionState, telemetryWithExp, undefined, promptOpts); +} + +export async function getGhostTextInternal( + accessor: ServicesAccessor, + textDocument: ITextDocument, + position: IPosition, + token?: CancellationToken +) { + return getGhostText(accessor, createCompletionState(textDocument, position), token, { opportunityId: 'opId' }); +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/relatedFiles.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/relatedFiles.ts new file mode 100644 index 0000000..0e788f6 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/relatedFiles.ts @@ -0,0 +1,42 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IIgnoreService } from '../../../../../../../platform/ignore/common/ignoreService'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsFileSystemService } from '../../fileSystem'; +import { ICompletionsLogTargetService } from '../../logger'; +import { TelemetryWithExp } from '../../telemetry'; +import { + RelatedFilesDocumentInfo, + RelatedFilesProvider, + RelatedFilesResponse, + RelatedFileTrait, +} from '../similarFiles/relatedFiles'; + +export class MockTraitsProvider extends RelatedFilesProvider { + constructor( + private readonly traits: RelatedFileTrait[] = [ + { name: 'testTraitName', value: 'testTraitValue' }, + { name: 'TargetFrameworks', value: 'net8' }, + { name: 'LanguageVersion', value: '12' }, + ], + @IInstantiationService instantiationService: IInstantiationService, + @IIgnoreService ignoreService: IIgnoreService, + @ICompletionsLogTargetService logTarget: ICompletionsLogTargetService, + @ICompletionsFileSystemService fileSystemService: ICompletionsFileSystemService, + ) { + super(instantiationService, ignoreService, logTarget, fileSystemService); + } + + async getRelatedFilesResponse( + docInfo: RelatedFilesDocumentInfo, + telemetryData: TelemetryWithExp + ): Promise<RelatedFilesResponse | undefined> { + return Promise.resolve({ + entries: [], + traits: this.traits, + }); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/prompt/test/repository.test.ts b/completions-sample-code/vscode-node/lib/src/prompt/test/repository.test.ts new file mode 100644 index 0000000..0139983 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/prompt/test/repository.test.ts @@ -0,0 +1,80 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import path from 'path'; +import { createLibTestingContext } from '../../test/context'; +import { makeFsUri } from '../../util/uri'; +import { extractRepoInfo } from '../repository'; +import { IInstantiationService } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; + +suite('Extract repo info tests', function () { + const baseFolder = { uri: makeFsUri(path.resolve(__dirname, '../../../../../../../../')) }; + + test('Extract repo info', async function () { + const accessor = createLibTestingContext().createTestingAccessor(); + const info = await extractRepoInfo(accessor, baseFolder.uri); + + assert.ok(info); + + // url and pathname get their own special treatment because they depend on how the repo was cloned. + const { url, pathname, repoId, ...repoInfo } = info; + + assert.deepStrictEqual(repoInfo, { + baseFolder, + hostname: 'github.com' + }); + assert.ok(repoId); + assert.deepStrictEqual( + { org: repoId.org, repo: repoId.repo, type: repoId.type }, + { org: 'microsoft', repo: 'vscode-copilot-chat', type: 'github' } + ); + assert.ok( + [ + 'git@github.com:microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat.git', + ].includes(url), + `url is ${url}` + ); + assert.ok(pathname.startsWith('/github/vscode-copilot-chat') || pathname.startsWith('/microsoft/vscode-copilot-chat')); + + assert.deepStrictEqual(await extractRepoInfo(accessor, 'file:///tmp/does/not/exist/.git/config'), undefined); + }); + + test('Extract repo info - Jupyter Notebook vscode-notebook-cell ', async function () { + const cellUri = baseFolder.uri.replace(/^file:/, 'vscode-notebook-cell:'); + assert.ok(cellUri.startsWith('vscode-notebook-cell:')); + const accessor = createLibTestingContext().createTestingAccessor(); + const instantiationService = accessor.get(IInstantiationService); + const info = await extractRepoInfo(accessor, cellUri); + + assert.ok(info); + + // url and pathname get their own special treatment because they depend on how the repo was cloned. + const { url, pathname, repoId, ...repoInfo } = info; + + assert.deepStrictEqual(repoInfo, { + baseFolder, + hostname: 'github.com' + }); + assert.ok(repoId); + assert.deepStrictEqual( + { org: repoId.org, repo: repoId.repo, type: repoId.type }, + { org: 'microsoft', repo: 'vscode-copilot-chat', type: 'github' } + ); + assert.ok( + [ + 'git@github.com:microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat.git', + ].includes(url), + `url is ${url}` + ); + assert.ok(pathname.startsWith('/github/vscode-copilot-chat') || pathname.startsWith('/microsoft/vscode-copilot-chat')); + + assert.deepStrictEqual(await instantiationService.invokeFunction(extractRepoInfo, 'file:///tmp/does/not/exist/.git/config'), undefined); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/snippy/compute.ts b/completions-sample-code/vscode-node/lib/src/snippy/compute.ts new file mode 100644 index 0000000..6b4fbd4 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/compute.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// This should be kept in sync with snippy at /pkg/fingerprint/compute.go#L20 +const SnippyLexemeRegex = new RegExp('[_\\p{L}\\p{Nd}]+|====+|----+|####+|////+|\\*\\*\\*\\*+|[\\p{P}\\p{S}]', 'gu'); +// This should be kept in sync with snippy at /pkg/fingerprint/settings.go#L108 +export const MinTokenLength = 65; + +export function lexemeLength(text: string) { + let i = 0; + let m: RegExpExecArray | null; + SnippyLexemeRegex.lastIndex = 0; + do { + m = SnippyLexemeRegex.exec(text); + if (m) { + i += 1; + } + + if (i >= MinTokenLength) { + break; + } + } while (m); + return i; +} + +/** Return the offset after the first `n` lexemes of `text`, counted in Snippy lexemes */ +function offsetFirstLexemes(text: string, n: number) { + let i = 0; + let m: RegExpExecArray | null; + SnippyLexemeRegex.lastIndex = 0; + do { + m = SnippyLexemeRegex.exec(text); + if (m) { + i += 1; + if (i >= n) { + return SnippyLexemeRegex.lastIndex; + } + } + } while (m); + // The whole text is less than n tokens + return text.length; +} + +/** Return the offset at the beginning of the last `n` lexemes of `text`, counted in Snippy lexemes */ +export function offsetLastLexemes(text: string, n: number) { + const textRev = text.split('').reverse().join(''); + const offsetRev = offsetFirstLexemes(textRev, n); + return textRev.length - offsetRev; +} + +export function hasMinLexemeLength(text: string) { + return lexemeLength(text) >= MinTokenLength; +} diff --git a/completions-sample-code/vscode-node/lib/src/snippy/connectionState.ts b/completions-sample-code/vscode-node/lib/src/snippy/connectionState.ts new file mode 100644 index 0000000..5370022 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/connectionState.ts @@ -0,0 +1,207 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsLogTargetService } from '../logger'; +import { getLastKnownEndpoints } from '../networkConfiguration'; +import { ICompletionsFetcherService } from '../networking'; +import { codeReferenceLogger } from './logger'; + +type ConnectionAPI = { + listen: (cb: () => void) => { dispose: () => void }; + setConnected: () => void; + setRetrying: () => void; + setDisconnected: () => void; + setDisabled: () => void; + enableRetry: (accessor: ServicesAccessor, initialTimeout?: number) => void; + isConnected: () => boolean; + isDisconnected: () => boolean; + isRetrying: () => boolean; + isDisabled: () => boolean; + isInitialWait: () => boolean; +}; + +type ConnectionState = { + connection: 'connected' | 'disconnected' | 'retry' | 'disabled'; + maxAttempts: number; + retryAttempts: number; + initialWait: boolean; +}; + +const InitialTimeout = 3000; +const BaseRetryTime = 2; +const MaxRetryTime = 256; +const MaxAttempts = Math.log(MaxRetryTime) / Math.log(BaseRetryTime) / BaseRetryTime; + +const state: ConnectionState = { + connection: 'disabled', + maxAttempts: MaxAttempts, + retryAttempts: 0, + initialWait: false, +}; + +let stateAPI: ConnectionAPI; +const handlers: Array<() => void> = []; + +function registerConnectionState(): ConnectionAPI { + if (stateAPI) { + return stateAPI; + } + + function subscribe(cb: () => void) { + handlers.push(cb); + return () => { + const index = handlers.indexOf(cb); + if (index !== -1) { + handlers.splice(index, 1); + } + }; + } + + function afterUpdateConnection() { + for (const handler of handlers) { + handler(); + } + } + + function updateConnection(status: ConnectionState['connection']) { + if (state.connection === status) { + return; + } + + state.connection = status; + afterUpdateConnection(); + } + + function isConnected() { + return state.connection === 'connected'; + } + + function isDisconnected() { + return state.connection === 'disconnected'; + } + + function isRetrying() { + return state.connection === 'retry'; + } + + function isDisabled() { + return state.connection === 'disabled'; + } + + function setConnected() { + updateConnection('connected'); + setInitialWait(false); + } + + function setDisconnected() { + updateConnection('disconnected'); + } + + function setRetrying() { + updateConnection('retry'); + } + + function setDisabled() { + updateConnection('disabled'); + } + + function setInitialWait(enabled: boolean) { + if (state.initialWait !== enabled) { + state.initialWait = enabled; + } + } + + function enableRetry(accessor: ServicesAccessor, initialTimeout = InitialTimeout) { + if (isRetrying()) { + return; + } + + setRetrying(); + setInitialWait(true); + void attemptToPing(accessor, initialTimeout); + } + + function isInitialWait() { + return state.initialWait; + } + + async function attemptToPing(accessor: ServicesAccessor, initialTimeout: number) { + const logTarget = accessor.get(ICompletionsLogTargetService); + const fetcher = accessor.get(ICompletionsFetcherService); + const instantiationService = accessor.get(IInstantiationService); + codeReferenceLogger.info(logTarget, `Attempting to reconnect in ${initialTimeout}ms.`); + + // Initial 3 second delay before attempting to reconnect to Snippy. + await timeout(initialTimeout); + setInitialWait(false); + + function succeedOrRetry(time: number) { + if (time > MaxRetryTime) { + codeReferenceLogger.info(logTarget, 'Max retry time reached, disabling.'); + setDisabled(); + return; + } + + const tryAgain = async () => { + state.retryAttempts = Math.min(state.retryAttempts + 1, MaxAttempts); + + try { + codeReferenceLogger.info(logTarget, `Pinging service after ${time} second(s)`); + const response = await fetcher.fetch( + new URL('_ping', instantiationService.invokeFunction(getLastKnownEndpoints)['origin-tracker']).href, + { + method: 'GET', + headers: { + 'content-type': 'application/json', + }, + } + ); + + if (response.status !== 200 || !response.ok) { + succeedOrRetry(time ** 2); + } else { + codeReferenceLogger.info(logTarget, 'Successfully reconnected.'); + setConnected(); + return; + } + } catch (e) { + succeedOrRetry(time ** 2); + } + }; + setTimeout(() => void tryAgain(), time * 1000); + } + + codeReferenceLogger.info(logTarget, 'Attempting to reconnect.'); + + succeedOrRetry(BaseRetryTime); + } + + const timeout = (ms: number) => { + return new Promise(resolve => setTimeout(resolve, ms)); + }; + + function listen(cb: () => void) { + const disposer = subscribe(cb); + return { dispose: disposer }; + } + + stateAPI = { + setConnected, + setDisconnected, + setRetrying, + setDisabled, + enableRetry, + listen, + isConnected, + isDisconnected, + isRetrying, + isDisabled, + isInitialWait, + }; + + return stateAPI; +} + +export const ConnectionState = registerConnectionState(); diff --git a/completions-sample-code/vscode-node/lib/src/snippy/constants.ts b/completions-sample-code/vscode-node/lib/src/snippy/constants.ts new file mode 100644 index 0000000..c647371 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/constants.ts @@ -0,0 +1,6 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export const OutputPaneShowCommand = 'codereferencing.showOutputPane2'; +export const FeatureName = 'code-referencing'; diff --git a/completions-sample-code/vscode-node/lib/src/snippy/errorCreator.ts b/completions-sample-code/vscode-node/lib/src/snippy/errorCreator.ts new file mode 100644 index 0000000..d852118 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/errorCreator.ts @@ -0,0 +1,65 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export type FormattedSnippyError = { kind: 'failure'; reason: string; code: number; msg: string; meta: object }; +export const ErrorReasons = { + BadArguments: 'BadArgumentsError', + Unauthorized: 'NotAuthorized', + NotFound: 'NotFoundError', + RateLimit: 'RateLimitError', + InternalError: 'InternalError', + ConnectionError: 'ConnectionError', + Unknown: 'UnknownError', +} as const; + +export const ErrorMessages = { + [ErrorReasons.Unauthorized]: + 'Invalid GitHub token. Please sign out from your GitHub account using VSCode UI and try again', + [ErrorReasons.InternalError]: + 'Internal error: matches to public code will not be detected. It is advised to disable Copilot completions until the service is reconnected.', + [ErrorReasons.RateLimit]: + `You've reached your quota and limit, code matching will be unavailable until the limit resets`, +}; + +export function getErrorType(code: number) { + if (code === 401) { + return ErrorReasons.Unauthorized; + } else if (code === 400) { + return ErrorReasons.BadArguments; + } else if (code === 404) { + return ErrorReasons.NotFound; + } else if (code === 429) { + return ErrorReasons.RateLimit; + } else if (code >= 500 && code < 600) { + return ErrorReasons.InternalError; + } else if (code >= 600) { + // internal error codes for reconnecting / fully disconnected state. open to changing. + // Separated because a 500 indicates a server error, but a 600 indicates the client is attempting + // to recover. + return ErrorReasons.ConnectionError; + } + + return ErrorReasons.Unknown; +} + +/** + * Helper method to combine a fetch response and a snippy error response into an + * object which conforms to our other error response interfaces. As seen in, e.g., extension/src/auth.ts. + * @param code HTTP status code + * @param msg + * @param meta Any additional data, typically an object + * @returns FormattedSnippyError + */ +export function createErrorResponse(code: number | string, msg: string, meta = {}) { + const reason = getErrorType(Number(code)); + const errorResponse: FormattedSnippyError = { + kind: 'failure', + reason, + code: Number(code), + msg, + meta, + }; + + return errorResponse; +} diff --git a/completions-sample-code/vscode-node/lib/src/snippy/handlePostInsertion.ts b/completions-sample-code/vscode-node/lib/src/snippy/handlePostInsertion.ts new file mode 100644 index 0000000..8428b91 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/handlePostInsertion.ts @@ -0,0 +1,153 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { Value } from '@sinclair/typebox/value'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsCitationManager } from '../citationManager'; +import { ICompletionsLogTargetService } from '../logger'; +import { ICompletionsTextDocumentManagerService } from '../textDocumentManager'; +import * as Snippy from './'; +import * as SnippyCompute from './compute'; +import { codeReferenceLogger } from './logger'; +import { MatchError } from './snippy.proto'; +import { snippyTelemetry } from './telemetryHandlers'; + +function isError(payload: unknown): payload is MatchError { + return Value.Check(MatchError, payload); +} + +async function snippyRequest<T>(accessor: ServicesAccessor, requestFn: () => T): Promise<ReturnType<typeof requestFn> | undefined> { + const instantiationService = accessor.get(IInstantiationService); + const res = await requestFn(); + + if (isError(res)) { + snippyTelemetry.handleSnippyNetworkError({ + instantiationService, + origin: String(res.code), + reason: res.reason, + message: res.msg, + }); + + return; + } + + return res; +} + +function isMatchError<T extends object>(response: T | MatchError): response is MatchError { + return 'kind' in response && response.kind === 'failure'; +} + +export async function fetchCitations(accessor: ServicesAccessor, uri: string, completionText: string, insertionOffset: number) { + const instantiationService = accessor.get(IInstantiationService); + const logTarget = accessor.get(ICompletionsLogTargetService); + const documentManager = accessor.get(ICompletionsTextDocumentManagerService); + const citationManager = accessor.get(ICompletionsCitationManager); + const insertionDoc = await documentManager.getTextDocument({ uri }); + + // If the match occurred in a file that no longer exists, bail. + if (!insertionDoc) { + codeReferenceLogger.debug(logTarget, `Expected document matching ${uri}, got nothing.`); + return; + } + + // The document text will include the completion at this point + const docText = insertionDoc.getText(); + + // If the document + the completion isn't long enough, we know we shouldn't call snippy + if (!SnippyCompute.hasMinLexemeLength(docText)) { + return; + } + + // If the document + the completion isn't long enough, we know we shouldn't call snippy + if (!SnippyCompute.hasMinLexemeLength(docText)) { + return; + } + + let potentialMatchContext = completionText; + + // In many cases, we will get completion that is shorter than 65 tokens, + // e.g. a single line or word completion. + // When a completion is too short, we should try and get the preceding tokens and + // pass that to snippy as part of the context. + if (!SnippyCompute.hasMinLexemeLength(completionText)) { + const textWithoutCompletion = docText.slice(0, insertionOffset); + const minLexemeStartOffset = SnippyCompute.offsetLastLexemes( + textWithoutCompletion, + SnippyCompute.MinTokenLength + ); + potentialMatchContext = docText.slice(minLexemeStartOffset, insertionOffset + completionText.length); + } + + // Depending on where in the document the suggestion was inserted, we may still not have enough context + // to detect a match. + if (!SnippyCompute.hasMinLexemeLength(potentialMatchContext)) { + return; + } + + const matchResponse = await instantiationService.invokeFunction(acc => snippyRequest(acc, () => Snippy.Match(acc, potentialMatchContext))); + + if (!matchResponse || isMatchError(matchResponse) || !matchResponse.snippets.length) { + // No match response from Snippy + codeReferenceLogger.info(logTarget, 'No match found'); + return; + } + + codeReferenceLogger.info(logTarget, 'Match found'); + + const { snippets } = matchResponse; + + const citationPromises = snippets.map(async snippet => { + const response = await instantiationService.invokeFunction(acc => snippyRequest(acc, () => Snippy.FilesForMatch(acc, { cursor: snippet.cursor }))); + + if (!response || isMatchError(response)) { + return; + } + + const files = response.file_matches; + const licenseStats = response.license_stats; + + return { + match: snippet, + files, + licenseStats, + }; + }); + + const citations = await Promise.all(citationPromises); + const filtered = citations.filter(c => c !== undefined); + // This shouldn't ever happen, but we should handle it nonetheless. + if (!filtered.length) { + return; + } + + for (const citation of filtered) { + const licensesSet = new Set(Object.keys(citation.licenseStats?.count ?? {})); + + if (licensesSet.has('NOASSERTION')) { + licensesSet.delete('NOASSERTION'); + licensesSet.add('unknown'); + } + + const allLicenses = Array.from(licensesSet).sort(); + + const offsetStart = insertionOffset; + const offsetEnd = insertionOffset + citation.match.matched_source.length; + + const start = insertionDoc.positionAt(offsetStart); + const end = insertionDoc.positionAt(offsetEnd); + await citationManager.handleIPCodeCitation({ + inDocumentUri: uri, + offsetStart, + offsetEnd, + version: insertionDoc.version, + location: { start, end }, + matchingText: potentialMatchContext, + details: allLicenses.map(license => ({ + license, + url: citation.match.github_url, + })), + }); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/snippy/index.ts b/completions-sample-code/vscode-node/lib/src/snippy/index.ts new file mode 100644 index 0000000..a2ba77a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/index.ts @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import type { IAbortSignal } from '../networking'; +import { assertShape } from '../util/typebox'; + +import { ICAPIClientService } from '../../../../../../platform/endpoint/common/capiClient'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import * as Network from './network'; +import * as Schema from './snippy.proto'; + +export async function Match(accessor: ServicesAccessor, source: string, signal?: IAbortSignal) { + const result = await Network.call<typeof Schema.MatchResponse>( + accessor, + accessor.get(ICAPIClientService).snippyMatchPath, + { + method: 'POST', + body: assertShape(Schema.MatchRequest, { source }), + }, + signal + ); + + const payload = assertShape(Schema.MatchResponse, result); + + return payload; +} + +export async function FilesForMatch(accessor: ServicesAccessor, { cursor }: Schema.FileMatchRequest, signal?: IAbortSignal) { + const result = await Network.call<typeof Schema.FileMatchResponse>( + accessor, + accessor.get(ICAPIClientService).snippyFilesForMatchPath, + { + method: 'POST', + body: assertShape(Schema.FileMatchRequest, { cursor }), + }, + signal + ); + + const payload = assertShape(Schema.FileMatchResponse, result); + + return payload; +} diff --git a/completions-sample-code/vscode-node/lib/src/snippy/logger.ts b/completions-sample-code/vscode-node/lib/src/snippy/logger.ts new file mode 100644 index 0000000..59fc49e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/logger.ts @@ -0,0 +1,8 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { Logger } from '../logger'; +import { FeatureName } from './constants'; + +export const codeReferenceLogger = new Logger(FeatureName); diff --git a/completions-sample-code/vscode-node/lib/src/snippy/network.ts b/completions-sample-code/vscode-node/lib/src/snippy/network.ts new file mode 100644 index 0000000..8fef984 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/network.ts @@ -0,0 +1,123 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CopilotToken, ICompletionsCopilotTokenManager } from '../auth/copilotTokenManager'; +import { editorVersionHeaders } from '../config'; +import { ICompletionsLogTargetService } from '../logger'; +import { getEndpointUrl } from '../networkConfiguration'; +import { ICompletionsFetcherService, type IAbortSignal, type Response } from '../networking'; +import { ConnectionState } from './connectionState'; +import { + createErrorResponse, + ErrorMessages, + ErrorReasons, + FormattedSnippyError, + getErrorType, +} from './errorCreator'; +import { codeReferenceLogger } from './logger'; +import { snippyTelemetry } from './telemetryHandlers'; + +type Config<Req> = { method: 'GET' } | { method: 'POST'; body: Req }; +type SnippyResponse<Res> = ({ kind: 'success' } & Res) | FormattedSnippyError; + +export async function call<Res, Req = unknown>( + accessor: ServicesAccessor, + endpoint: string, + config: Config<Req>, + signal?: IAbortSignal +): Promise<SnippyResponse<Res>> { + let token: CopilotToken; + const logTarget = accessor.get(ICompletionsLogTargetService); + const instantiationService = accessor.get(IInstantiationService); + const tokenManager = accessor.get(ICompletionsCopilotTokenManager); + try { + token = tokenManager.token ?? await tokenManager.getToken(); + } catch (e) { + ConnectionState.setDisconnected(); + return createErrorResponse(401, ErrorMessages[ErrorReasons.Unauthorized]); + } + + codeReferenceLogger.info(logTarget, `Calling ${endpoint}`); + + if (ConnectionState.isRetrying()) { + return createErrorResponse(600, 'Attempting to reconnect to the public code matching service.'); + } + + if (ConnectionState.isDisconnected()) { + return createErrorResponse(601, 'The public code matching service is offline.'); + } + + let res: InstanceType<typeof Response>; + try { + res = await instantiationService.invokeFunction(acc => acc.get(ICompletionsFetcherService).fetch(getEndpointUrl(acc, token, 'origin-tracker', endpoint), { + method: config.method, + body: config.method === 'POST' ? JSON.stringify(config.body) : undefined, + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${token.token}`, + ...editorVersionHeaders(acc), + }, + signal, + })); + } catch (e) { + instantiationService.invokeFunction(ConnectionState.enableRetry); + return createErrorResponse(602, 'Network error detected. Check your internet connection.'); + } + + let payload; + try { + payload = await res.json(); + } catch (e) { + const message = (e as Error).message; + snippyTelemetry.handleUnexpectedError({ + instantiationService, + origin: 'snippyNetwork', + reason: message, + }); + throw e; + } + + if (res.ok) { + return { + kind: 'success', + ...(payload as Res), + }; + } + const errorPayload = { + ...(payload as FormattedSnippyError), + code: Number(res.status), + }; + + /** + * Snippy will always respond with a 200, unless: + * + * - the request is malformed + * - the user is not authorized. + * - the server is down + */ + const { code, msg, meta } = errorPayload; + const formattedCode = Number(code); + const errorTypeFromCode = getErrorType(formattedCode); + const fallbackMsg = msg || 'unknown error'; + switch (errorTypeFromCode) { + case ErrorReasons.Unauthorized: { + return createErrorResponse(code, ErrorMessages[ErrorReasons.Unauthorized], meta); + } + case ErrorReasons.BadArguments: { + return createErrorResponse(code, fallbackMsg, meta); + } + case ErrorReasons.RateLimit: { + instantiationService.invokeFunction(acc => ConnectionState.enableRetry(acc, 60 * 1000)); + return createErrorResponse(code, ErrorMessages.RateLimitError, meta); + } + case ErrorReasons.InternalError: { + instantiationService.invokeFunction(acc => ConnectionState.enableRetry(acc)); + return createErrorResponse(code, ErrorMessages[ErrorReasons.InternalError], meta); + } + default: { + return createErrorResponse(code, fallbackMsg, meta); + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/snippy/snippy.proto.ts b/completions-sample-code/vscode-node/lib/src/snippy/snippy.proto.ts new file mode 100644 index 0000000..6b9e533 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/snippy.proto.ts @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** + * File inspired by snippy at /proto/snippy.proto + */ + +import { Static, Type } from '@sinclair/typebox'; + +export const MatchError = Type.Object({ + kind: Type.Literal('failure'), + reason: Type.String(), + code: Type.Number(), + msg: Type.String(), + meta: Type.Optional(Type.Any()), +}); +export type MatchError = Static<typeof MatchError>; + +const Snippet = Type.Object({ + matched_source: Type.String(), + occurrences: Type.String(), + capped: Type.Boolean(), + cursor: Type.String(), + github_url: Type.String(), +}); +type Snippet = Static<typeof Snippet>; + +export const MatchRequest = Type.Object({ + source: Type.String(), +}); +export type MatchRequest = Static<typeof MatchRequest>; + +const MatchSuccess = Type.Object({ + snippets: Type.Array(Snippet), +}); +type MatchSuccess = Static<typeof MatchSuccess>; +export const MatchResponse = Type.Union([ + // Snippet type + MatchSuccess, + // Error type + MatchError, +]); +export type MatchResponse = Static<typeof MatchResponse>; + +export const FileMatchRequest = Type.Object({ + cursor: Type.String(), +}); +export type FileMatchRequest = Static<typeof FileMatchRequest>; + +const FileMatch = Type.Object({ + commit_id: Type.String(), + license: Type.String(), + nwo: Type.String(), + path: Type.String(), + url: Type.String(), +}); +type FileMatch = Static<typeof FileMatch>; + +const PageInfo = Type.Object({ + has_next_page: Type.Boolean(), + cursor: Type.String(), +}); + +const LicenseStats = Type.Object({ + count: Type.Record(Type.String(), Type.String()), +}); +type LicenseStats = Static<typeof LicenseStats>; + +const FileMatchSuccess = Type.Object({ + file_matches: Type.Array(FileMatch), + page_info: PageInfo, + license_stats: LicenseStats, +}); +type FileMatchSuccess = Static<typeof FileMatchSuccess>; +export const FileMatchResponse = Type.Union([FileMatchSuccess, MatchError]); +export type FileMatchResponse = Static<typeof FileMatchResponse>; diff --git a/completions-sample-code/vscode-node/lib/src/snippy/telemetryHandlers.ts b/completions-sample-code/vscode-node/lib/src/snippy/telemetryHandlers.ts new file mode 100644 index 0000000..b936a92 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/telemetryHandlers.ts @@ -0,0 +1,127 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsLogTargetService } from '../logger'; +import { telemetry, TelemetryData, telemetryError } from '../telemetry'; +import { codeReferenceLogger } from './logger'; + +export type TelemetryActor = 'user' | 'system'; + +type Base = { + instantiationService: IInstantiationService; +}; +type MatchUIDetails = Base & { actor: TelemetryActor }; + +type PostInsertionErrorDetails = Base & { + origin: string; + reason: string; +}; + +type SnippyNetworkErrorDetails = PostInsertionErrorDetails & { + message: string; +}; + +// Check for valid http status code format. We use 6xx internally. +const statusCodeRe = /^[1-6][0-9][0-9]$/; +// Look for capital letters followed by lowercase letters. +const capitalsRe = /([A-Z][a-z]+)/; +const NAMESPACE = 'code_referencing'; + +class CodeQuoteTelemetry { + constructor(protected readonly baseKey: string) { } + buildKey(...keys: string[]) { + return [NAMESPACE, this.baseKey, ...keys].join('.'); + } +} + +class CopilotOutputLogTelemetry extends CodeQuoteTelemetry { + constructor() { + super('github_copilot_log'); + } + + handleOpen({ instantiationService }: Base) { + const key = this.buildKey('open', 'count'); + const data = TelemetryData.createAndMarkAsIssued(); + instantiationService.invokeFunction(telemetry, key, data); + } + + handleFocus({ instantiationService }: Base) { + const data = TelemetryData.createAndMarkAsIssued(); + const key = this.buildKey('focus', 'count'); + instantiationService.invokeFunction(telemetry, key, data); + } + + handleWrite({ instantiationService }: Base) { + const data = TelemetryData.createAndMarkAsIssued(); + const key = this.buildKey('write', 'count'); + instantiationService.invokeFunction(telemetry, key, data); + } +} + +export const copilotOutputLogTelemetry = new CopilotOutputLogTelemetry(); + +class MatchNotificationTelemetry extends CodeQuoteTelemetry { + constructor() { + super('match_notification'); + } + + handleDoAction({ instantiationService, actor }: MatchUIDetails) { + const data = TelemetryData.createAndMarkAsIssued({ actor }); + const key = this.buildKey('acknowledge', 'count'); + instantiationService.invokeFunction(telemetry, key, data); + } + + handleDismiss({ instantiationService, actor }: MatchUIDetails) { + const data = TelemetryData.createAndMarkAsIssued({ actor }); + const key = this.buildKey('ignore', 'count'); + instantiationService.invokeFunction(telemetry, key, data); + } +} + +export const matchNotificationTelemetry = new MatchNotificationTelemetry(); + +class SnippyTelemetry extends CodeQuoteTelemetry { + constructor() { + super('snippy'); + } + + handleUnexpectedError({ instantiationService, origin, reason }: PostInsertionErrorDetails) { + const data = TelemetryData.createAndMarkAsIssued({ origin, reason }); + instantiationService.invokeFunction(telemetryError, this.buildKey('unexpectedError'), data); + } + + handleCompletionMissing({ instantiationService, origin, reason }: PostInsertionErrorDetails) { + const data = TelemetryData.createAndMarkAsIssued({ origin, reason }); + instantiationService.invokeFunction(telemetryError, this.buildKey('completionMissing'), data); + } + + handleSnippyNetworkError({ instantiationService, origin, reason, message }: SnippyNetworkErrorDetails) { + if (!origin.match(statusCodeRe)) { + instantiationService.invokeFunction(acc => codeReferenceLogger.debug(acc.get(ICompletionsLogTargetService), 'Invalid status code, not sending telemetry', { origin })); + return; + } + + // reason is a string like "SnippyNetworkError". We want to format it to use underscores, which + // is the standard for Copilot telemetry keys. + const errorType = reason + .split(capitalsRe) + .filter(part => Boolean(part)) + .join('_') + .toLowerCase(); + const data = TelemetryData.createAndMarkAsIssued({ message }); + instantiationService.invokeFunction(telemetryError, this.buildKey(errorType, origin), data); + } +} + +export const snippyTelemetry = new SnippyTelemetry(); + +/** @public KEEPING FOR TESTS */ +export class NoopTelemetryReporter extends CodeQuoteTelemetry { + constructor(baseKey = '') { + super(baseKey); + } + telemetry(...args: Parameters<typeof telemetry>) { } + telemetryError(...args: Parameters<typeof telemetryError>) { } +} diff --git a/completions-sample-code/vscode-node/lib/src/snippy/test/compute.test.ts b/completions-sample-code/vscode-node/lib/src/snippy/test/compute.test.ts new file mode 100644 index 0000000..4c44b7a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/test/compute.test.ts @@ -0,0 +1,45 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import * as SnippyCompute from '../../snippy/compute'; + +const testMatchSource = + 'function calculateDaysBetweenDates(begin, end) {\n var oneDay = 24*60*60*1000; // hours*minutes*seconds*milliseconds\n var firstDate = new Date(begin);\n var secondDate = new Date(end);\n\n return Math.round(Math.abs((firstDate.getTime() - secondDate.getTime())/(oneDay)));\n}'; + +suite('Compute', function () { + const testCases = [ + { input: 'const', expected: 1 }, + { input: 'const foo = "bar";', expected: 7 }, + { + input: `for (var i = 1; i <= 100; i++) { + if (i % 15 == 0) { + console.log("FizzBuzz"); + } else if (i % 3 == 0) { + console.log("Fizz"); + } else if (i % 5 == 0) { + console.log("Buzz"); + } else { + console.log(i); + } + }`, + expected: 65, + }, + ]; + + test('lexemeLength returns the number of lexemes in a given string', function () { + for (const { input, expected } of testCases) { + assert.strictEqual(SnippyCompute.lexemeLength(input), expected); + } + }); + + test(`lexemeLength returns at most ${SnippyCompute.MinTokenLength} lexemes`, function () { + assert.strictEqual(SnippyCompute.lexemeLength(testMatchSource), SnippyCompute.MinTokenLength); + }); + + test(`hasMinLexemeLength returns true if the string has at least ${SnippyCompute.MinTokenLength} lexemes`, function () { + assert.strictEqual(SnippyCompute.hasMinLexemeLength(testMatchSource), true); + assert.strictEqual(SnippyCompute.hasMinLexemeLength(`const foo = 'test'`), false); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/snippy/test/network.test.ts b/completions-sample-code/vscode-node/lib/src/snippy/test/network.test.ts new file mode 100644 index 0000000..61bcdfc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/snippy/test/network.test.ts @@ -0,0 +1,204 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import * as Sinon from 'sinon'; +import { ServicesAccessor } from '../../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsCopilotTokenManager } from '../../auth/copilotTokenManager'; +import { + ConfigKey, ICompletionsConfigProvider, InMemoryConfigProvider +} from '../../config'; +import { ICompletionsFetcherService, Response } from '../../networking'; +import { ConnectionState } from '../../snippy/connectionState'; +import { ErrorMessages, ErrorReasons, FormattedSnippyError } from '../../snippy/errorCreator'; +import * as Network from '../../snippy/network'; +import { createLibTestingContext } from '../../test/context'; +import { FakeFetcher, createFakeJsonResponse } from '../../test/fetcher'; + +const testEndpoints: Record< + string, + { response: Record<string, string>; status: number; expected: Record<string, string> } +> = { + '400': { + status: 400, + response: { code: 'invalid_argument', msg: 'source too short' }, + expected: { + reason: ErrorReasons.BadArguments, + msg: 'source too short', + }, + }, + '401': { + status: 401, + response: { error: 'unauthorized' }, + expected: { + reason: ErrorReasons.Unauthorized, + msg: ErrorMessages[ErrorReasons.Unauthorized], + }, + }, + '402': { + status: 402, + response: { code: 'payment required', msg: '' }, + expected: { + reason: ErrorReasons.Unknown, + msg: 'unknown error', + }, + }, + '404': { + status: 404, + response: { code: 'bad_route', msg: 'no handler for path' }, + expected: { + reason: ErrorReasons.NotFound, + msg: 'no handler for path', + }, + }, + '429': { + status: 429, + response: { code: 'rate_limited', msg: 'rate limit' }, + expected: { + reason: ErrorReasons.RateLimit, + msg: ErrorMessages[ErrorReasons.RateLimit], + }, + }, + '500': { + status: 500, + response: { error: 'Internal error' }, + expected: { + reason: ErrorReasons.InternalError, + msg: ErrorMessages[ErrorReasons.InternalError], + }, + }, + '503': { + status: 503, + response: { error: 'Network error' }, + expected: { + reason: ErrorReasons.InternalError, + msg: ErrorMessages[ErrorReasons.InternalError], + }, + }, +}; + +class SnippyFetcher extends FakeFetcher { + constructor() { + super(); + } + + fetch(url: string): Promise<Response> { + const endpoint = url.split('/').pop()!; + const testCase = testEndpoints[endpoint] || testEndpoints['404']; + + return Promise.resolve(createFakeJsonResponse(testCase.status, testCase.response)); + } +} + +suite('snippy network primitive', function () { + let accessor: ServicesAccessor; + let originalConfigProvider: InMemoryConfigProvider; + + setup(function () { + const serviceCollection = createLibTestingContext(); + serviceCollection.define(ICompletionsFetcherService, new SnippyFetcher()); + accessor = serviceCollection.createTestingAccessor(); + originalConfigProvider = accessor.get(ICompletionsConfigProvider) as InMemoryConfigProvider; + }); + + teardown(function () { + ConnectionState.setConnected(); + originalConfigProvider.clearOverrides(); + }); + + suite('error handling', function () { + test.skip('should return a 401 error object when token is invalid', async function () { + //setStaticSessionTokenManager(ctx, undefined); + const tokenManager = accessor.get(ICompletionsCopilotTokenManager); + tokenManager.resetToken(); + + const response: FormattedSnippyError = await Network.call(accessor, '', { method: 'GET' }); + + assert.strictEqual(response.kind, 'failure'); + assert.strictEqual(response.code, 401); + assert.strictEqual(response.reason, ErrorReasons.Unauthorized); + assert.strictEqual(response.msg, ErrorMessages[ErrorReasons.Unauthorized]); + }); + test('should return a 600 error object when connection is retrying', async function () { + ConnectionState.setRetrying(); + + const response: FormattedSnippyError = await Network.call(accessor, '', { method: 'GET' }); + + assert.strictEqual(response.kind, 'failure'); + assert.strictEqual(response.code, 600); + assert.strictEqual(response.reason, ErrorReasons.ConnectionError); + assert.strictEqual(response.msg, 'Attempting to reconnect to the public code matching service.'); + }); + + test('should return a 601 error object when connection is offline', async function () { + ConnectionState.setDisconnected(); + + const response: FormattedSnippyError = await Network.call(accessor, '', { method: 'GET' }); + + assert.strictEqual(response.kind, 'failure'); + assert.strictEqual(response.code, 601); + assert.strictEqual(response.reason, ErrorReasons.ConnectionError); + assert.strictEqual(response.msg, 'The public code matching service is offline.'); + }); + + test('should return the expect payload for various error codes', async function () { + const testCases = Object.entries(testEndpoints); + // Internal errors put CodeQuote into retry mode, so we need to stub that behavior out. + const stub = Sinon.stub(ConnectionState, 'enableRetry').callsFake(() => { }); + + for (const [endpoint, data] of testCases) { + const response: FormattedSnippyError = await Network.call(accessor, endpoint, { method: 'GET' }); + + assert.strictEqual(response.kind, 'failure'); + assert.strictEqual(response.code, data.status); + assert.strictEqual(response.reason, data.expected.reason); + assert.strictEqual(response.msg, data.expected.msg); + } + + stub.restore(); + }); + }); + + suite('`call` behavior', function () { + const sandbox = Sinon.createSandbox(); + let networkStub: Sinon.SinonStub<Parameters<ICompletionsFetcherService['fetch']>>; + + setup(function () { + networkStub = Sinon.stub(accessor.get(ICompletionsFetcherService), 'fetch'); + networkStub.returns(Promise.resolve(createFakeJsonResponse(200, '{}'))); + }); + + teardown(function () { + sandbox.restore(); + }); + + test('uses alternative endpoint when specified', async function () { + const overrides = new Map<string, unknown>(); + const domainOverride = 'https://fake.net.biz/'; + overrides.set(ConfigKey.DebugSnippyOverrideUrl, domainOverride); + + originalConfigProvider.setOverrides(overrides); + + await Network.call(accessor, '', { method: 'GET' }); + + assert.ok(networkStub.getCall(0).args[0].startsWith(domainOverride)); + }); + + test('uses the correct snippy twirp endpoint', async function () { + await Network.call(accessor, 'endpoint/snippy', { method: 'GET' }); + const url = networkStub.getCall(0).args[0]; + assert.ok(url.includes('endpoint/snippy')); + }); + + test('supplies editor information to snippy', async function () { + await Network.call(accessor, '', { method: 'GET' }); + + const headers = networkStub.getCall(0).args[1].headers ?? {}; + const headerKeys = Object.keys(headers); + + assert.ok(headerKeys.includes('Editor-Version')); + assert.ok(headerKeys.includes('Editor-Plugin-Version')); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/anomalyDetection.ts b/completions-sample-code/vscode-node/lib/src/suggestions/anomalyDetection.ts new file mode 100644 index 0000000..6d4a02e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/anomalyDetection.ts @@ -0,0 +1,79 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** Sometimes the model gets caught in repeating a simplistic and unhelpful pattern + * This file provides functionality to check whether this might be the case + */ + +interface RepetitionConfig { + max_token_sequence_length: number; + last_tokens_to_consider: number; +} + +const configs: RepetitionConfig[] = [ + // in case of single token, 10 repetitions is too much already: + { max_token_sequence_length: 1, last_tokens_to_consider: 10 }, + // if the last 30 tokens are a repeat of up to 10 tokens, then it's a pattern: + { max_token_sequence_length: 10, last_tokens_to_consider: 30 }, + // if the pattern is very long, it needs to account for a long stretch so we can be sure + { max_token_sequence_length: 20, last_tokens_to_consider: 45 }, + { max_token_sequence_length: 30, last_tokens_to_consider: 60 }, +]; + +/** + * Return whether the given token array ends in a repetition of a pattern. + * Controlling the necessary pattern length is set in the configs array. + */ +export function isRepetitive(tokens: readonly string[]): boolean { + const tokensBackwards = tokens.slice(); + tokensBackwards.reverse(); + return ( + isRepeatedPattern(tokensBackwards) || + isRepeatedPattern(tokensBackwards.filter(token => token.trim().length > 0)) + ); +} + +/** + * Determine whether the given array or string starts with the repetition of a pattern, + * according to one of the predefined configs. + */ +function isRepeatedPattern<T>(s: ArrayLike<T>): boolean { + const prefix = kmp_prefix_function(s); + for (const config of configs) { + if (s.length < config.last_tokens_to_consider) { + continue; + } + // This is the smallest number of characters that one may shift `s` so that it + // overlaps with itself. That is also the smallest length of a repeated + // pattern that makes up `s`, where the last repetition is possibly truncated. + const patternLength = config.last_tokens_to_consider - 1 - prefix[config.last_tokens_to_consider - 1]; + if (patternLength <= config.max_token_sequence_length) { + return true; + } + } + return false; +} + +/** Return the Knuth-Morris-Pratt prefix function pi. + * For each i=0,..,.s.length-1, then + * pi[i] = max(j < i, s.slice(0,i+1).beginsWith(s.slice(0, j+1))) + * (note pi[0] = -1 by this definition) + * Adapted from + * Introduction to Algorithms, 3rd edition, by Thomas H. Cormen, et al. + */ +function kmp_prefix_function<T>(s: ArrayLike<T>): number[] { + const pi = Array<number>(s.length).fill(0); + pi[0] = -1; + let k = -1; + for (let q = 1; q < s.length; q++) { + while (k >= 0 && s[k + 1] !== s[q]) { + k = pi[k]; + } + if (s[k + 1] === s[q]) { + k++; + } + pi[q] = k; + } + return pi; +} diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/editDistance.ts b/completions-sample-code/vscode-node/lib/src/suggestions/editDistance.ts new file mode 100644 index 0000000..4f1e8db --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/editDistance.ts @@ -0,0 +1,265 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +interface Alignment { + distance: number; + startOffset: number; + endOffset: number; +} + +/** + * Computes the best alignment, under edit-distance, of placing `needle` within + * `haystack`. These may be strings or arrays. + * + * In other words, the entirety of `needle` will count towards the distance, + * while only the sub-range within `haystack` corresponding to the best match + * will be included. This means `editDistance(a, b) !== editDistance(b, a)` in + * general. + * + * If `needle` and `haystack` are strings, the distance is in UTF-16 code units. + * For instance, an emoji inserted in `needle` will increase the distance by 2, + * while an ASCII character will increase it only by one. + * + * @param haystack The big string or array within which the needle should match + * @param needle The small string or array to match + * @param compare An optional comparison operator for the elements of `haystack` + * and `needle`. It should return a "cost" for substituting a given element of + * `haystack` with a given element of `needle`. If these elements are equal then + * `compare` should return 0. The indices of these elements are also given to + * compare. + * + * @returns An alignment of the best match possible, with offsets within + * `haystack`. + */ +export function editDistance<E, T extends string | E[]>( + haystack: T, + needle: T, + compare: ( + haystackElem: (typeof haystack)[number], + needleElem: (typeof needle)[number], + haystackIndex: number, + needleIndex: number + ) => number = (h, n) => (h === n ? 0 : 1) +): Alignment { + if (needle.length === 0 || haystack.length === 0) { return { distance: needle.length, startOffset: 0, endOffset: 0 }; } + let curRow = new Array<number>(needle.length + 1).fill(0); + let curStart = new Array<number>(needle.length + 1).fill(0); + let prevRow = new Array<number>(haystack.length + 1).fill(0); + let prevStart = new Array<number>(haystack.length + 1).fill(0); + // Initialise the alignment of needle inside haystack + let c = needle[0]; + for (let i = 0; i < haystack.length + 1; i++) { + if (i === 0) { curRow[i] = 1; } + else { curRow[i] = compare(haystack[i - 1], c, i - 1, 0); } + // We record the starting offset as 0 in two distinct cases: + // - At least one char of needle is inserted left of haystack + // - 0th char of needle = or subst. 0'th char of haystack + curStart[i] = i > 0 ? i - 1 : 0; + } + // Iterate over the rest of needle + for (let j = 1; j < needle.length; j++) { + // Set curRow to prevRow, and reuse the prevRow allocation for this + // iteration (its contents will be entirely overwritten). + let swap = prevRow; + prevRow = curRow; + curRow = swap; + swap = prevStart; + prevStart = curStart; + curStart = swap; + + c = needle[j]; + curRow[0] = j + 1; // All chars of needle inserted before haystack. + // Note: curStart[0] = 0 is invariant + for (let i = 1; i < haystack.length + 1; i++) { + // What happens to the j'th char of needle + const inserted = 1 + prevRow[i]; // inserted after i'th char of haystack + const deleted = 1 + curRow[i - 1]; // deleted after i'th char of haystack + const substituted = compare(haystack[i - 1], c, i - 1, j) + prevRow[i - 1]; // substituted w. i'th char of haystack + curRow[i] = Math.min(deleted, inserted, substituted); + if (curRow[i] === substituted) { + curStart[i] = prevStart[i - 1]; + } else if (curRow[i] === inserted) { + curStart[i] = prevStart[i]; + } else { + curStart[i] = curStart[i - 1]; + } + } + } + + // Find the best matching end-offset + let best = 0; + for (let i = 0; i < haystack.length + 1; i++) { + if (curRow[i] < curRow[best]) { best = i; } + } + return { distance: curRow[best], startOffset: curStart[best], endOffset: best }; +} + +type LexDictionary = Map<string, number>; + +interface LexGenerator { + (s: string): Generator<string, void, unknown>; +} + +export function emptyLexDictionary(): LexDictionary { + return new Map(); +} + +export function reverseLexDictionary(d: LexDictionary): string[] { + const lookup = new Array<string>(d.size); + for (const [lexeme, idx] of d) { + lookup[idx] = lexeme; + } + return lookup; +} + +/** + * A simple lex generator. + * A lexeme is one of the following three: + * 1. A sequence of letters, numbers, _ and - + * 2. A sequence of spaces + * 3. Any other single Unicode code point + */ +export function* lexGeneratorWords(s: string): Generator<string, void, unknown> { + let buffer = ''; + enum State { + Word, + Space, + Other, + } + let state: State = State.Word; + for (const c of s) { + let newState: State; + if (/(\p{L}|\p{Nd}|_)/u.test(c)) { newState = State.Word; } + else if (c === ' ') { newState = State.Space; } + else { newState = State.Other; } + if (newState === state && newState !== State.Other) { + buffer += c; + } else { + if (buffer.length > 0) { yield buffer; } + buffer = c; + state = newState; + } + } + if (buffer.length > 0) { yield buffer; } +} + +/** + * Convert a string into an array of lexeme ids, as defined by a lexeme dictionary. + * + * Lexemes not already in the dictionary will be added with a fresh key. Hence, + * this function can be called with an `emptyLexDictionary()`. + * + * @param s The string to convert + * @param lexDictionary The dictionary to begin with + * @param lexGenerator The generator to use to convert `s` into a stream of + * substring lexemes + * @param lexFilter Keep only lexemes satisfying this conditional + * + * @returns Pair containing: + * - an array of (lexeme ids, lexeme starting offset within `s`), + * - the updated dictionary. + */ +export function lexicalAnalyzer( + s: string, + d: LexDictionary, + lexGenerator: LexGenerator, + lexFilter: (lexeme: string) => boolean +): [[number, number][], LexDictionary] { + const lexed = [] as [number, number][]; + let offset = 0; + for (const lexeme of lexGenerator(s)) { + if (lexFilter(lexeme)) { + if (!d.has(lexeme)) { d.set(lexeme, d.size); } + lexed.push([d.get(lexeme)!, offset]); + } + offset += lexeme.length; + } + return [lexed, d]; +} + +function notSingleSpace(s: string): boolean { + return s !== ' '; +} + +interface LexAlignment { + lexDistance: number; + startOffset: number; // offsets in utf-16 code units + endOffset: number; + haystackLexLength: number; + needleLexLength: number; +} + +/** + * Computes the best alignment, under edit-distance, of placing the lexemes of + * `needle` within those of `haystack`. + * + * More precisely, we compute the lex tokens of `needle` and `haystack` under + * the same dictionary, and then align these by their edit distance using + * `editDistance`. We then translate the offsets in the lex-match-alignment back + * to character offsets. + * + * @param haystack The big string within which the needle should match + * @param needle The small string to match + * @param lexGenerator Generator which chops up a string into lexemes + * @param lexFilter Keep only lexemes that return true on this function + * + * @returns An alignment of the best match possible, with offsets within + * `haystack`. + */ +export function lexEditDistance( + haystack: string, + needle: string, + lexGenerator: LexGenerator = lexGeneratorWords +): LexAlignment { + const [haystackLexed, d] = lexicalAnalyzer(haystack, emptyLexDictionary(), lexGenerator, notSingleSpace); + const [needleLexed, dBoth] = lexicalAnalyzer(needle, d, lexGenerator, notSingleSpace); + // Special case for empty haystack or needle (or either consisting of single space) + if (needleLexed.length === 0 || haystackLexed.length === 0) { + return { + lexDistance: needleLexed.length, + startOffset: 0, + endOffset: 0, + haystackLexLength: haystackLexed.length, + needleLexLength: needleLexed.length, + }; + } + // Align the lexed strings + // Take special care to not add cost if first lexeme of needle is postfix of + // lexeme in haystack, or last lexeme of needle is prefix of lexeme in + // haystack + const lookupId = reverseLexDictionary(dBoth); + const needleLexedLength = needleLexed.length; + const needleFirst = lookupId[needleLexed[0][0]]; + const needleLast = lookupId[needleLexed[needleLexedLength - 1][0]]; + function compare(hLexId: number, nLexId: number, hIndex: number, nIndex: number) { + if (nIndex === 0 || nIndex === needleLexedLength - 1) { + const haystackLexeme = lookupId[haystackLexed[hIndex][0]]; + return (nIndex === 0 && haystackLexeme.endsWith(needleFirst)) || + (nIndex === needleLexedLength - 1 && haystackLexeme.startsWith(needleLast)) + ? 0 + : 1; + } else { + return hLexId === nLexId ? 0 : 1; + } + } + const alignment = editDistance( + haystackLexed.map(x => x[0]), + needleLexed.map(x => x[0]), + compare + ); + // Convert the lexeme offsets in alignment to character offsets + const startOffset = haystackLexed[alignment.startOffset][1]; + let endOffset = + alignment.endOffset < haystackLexed.length ? haystackLexed[alignment.endOffset][1] : haystack.length; + // Account for a possible filtered-out single-space lexeme at end of match + if (endOffset > 0 && haystack[endOffset - 1] === ' ') { --endOffset; } + + return { + lexDistance: alignment.distance, + startOffset, + endOffset, + haystackLexLength: haystackLexed.length, + needleLexLength: needleLexed.length, + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/partialSuggestions.ts b/completions-sample-code/vscode-node/lib/src/suggestions/partialSuggestions.ts new file mode 100644 index 0000000..571e23a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/partialSuggestions.ts @@ -0,0 +1,37 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// Copy of https://github.com/microsoft/vscode/blob/969b5714b4fc54992801dceefc3269ce4e07f8f7/src/vscode-dts/vscode.proposed.inlineCompletionsAdditions.d.ts#L75 +// to avoid dependencies to vscode from lib +export enum PartialAcceptTriggerKind { + Unknown = 0, + Word = 1, + Line = 2, + Suggest = 3, +} + +type CompletionType = 'partial' | 'full'; + +export type SuggestionStatus = { + compType: CompletionType; + acceptedLength: number; + acceptedLines: number; // Number of lines accepted in the current completion, used for partial acceptance +}; + +export function computeCompCharLen(suggestionStatus: SuggestionStatus, completionText: string): number { + return suggestionStatus.compType === 'partial' ? suggestionStatus.acceptedLength : completionText.length; +} + +export function countLines(text: string): number { + if (text.length === 0) { return 0; } + + return text.split('\n').length; +} + +export function computeCompletionText(completionText: string, suggestionStatus: SuggestionStatus): string { + if (suggestionStatus.compType === 'partial') { + return completionText.substring(0, suggestionStatus.acceptedLength); + } + return completionText; +} diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/suggestions.ts b/completions-sample-code/vscode-node/lib/src/suggestions/suggestions.ts new file mode 100644 index 0000000..c075d2f --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/suggestions.ts @@ -0,0 +1,235 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// General utility functions for all kinds of suggestions (Ghost Text, Open Copilot) + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { getBlockCloseToken } from '../../../prompt/src/parse'; +import { ICompletionsLogTargetService, Logger } from '../logger'; +import { APIChoice } from '../openai/openai'; +import { TelemetryData, TelemetryStore, telemetry } from '../telemetry'; +import { IPosition, TextDocumentContents } from '../textDocument'; +import { isRepetitive } from './anomalyDetection'; + +/** + * To avoid double-closing blocks (#272), maybe snip a trailing block-close token + * from the given completion. + * + * We check whether the completion ends with a block-close token, and the next line + * after the cursor starts with that same token at the same indentation. If so, + * we snip. + */ +function maybeSnipCompletion(accessor: ServicesAccessor, doc: TextDocumentContents, position: IPosition, completion: string): string { + // Default to `}` for block closing token + let blockCloseToken = '}'; + + //TODO: This should be properly handled in promptlib (in `getBlockCloseToken`) + //but we don't want to change it before Universe. + try { + blockCloseToken = getBlockCloseToken(doc.detectedLanguageId) ?? '}'; + } catch (e) { + // Ignore errors + } + + return maybeSnipCompletionImpl( + { getLineText: lineIdx => doc.lineAt(lineIdx).text, getLineCount: () => doc.lineCount }, + position, + completion, + blockCloseToken + ); +} + +export interface ILines { + getLineText(lineIdx: number): string; + getLineCount(): number; +} + +export function maybeSnipCompletionImpl( + doc: ILines, + position: IPosition, + completion: string, + blockCloseToken: string +): string { + // if the last lines of the completion are just indented block close tokens (e.g. `\t}\n}`), + // and if these lines exactly match the lines of the document after the insertion position (ignoring empty lines in both the document and the completion), + // these lines are removed from the completion. + // Additionally, the last line of the completion can be a prefix of a line in the model. + // Thus, if `\tif (true) {\n\t}` is suggested and the next line of the doc is `\t} else {`, only `if (true) {` will be suggested. + + const completionLinesInfo = splitByNewLine(completion); + const completionLines = completionLinesInfo.lines; + if (completionLines.length === 1) { + return completion; + } + + for (let completionLineStartIdx = 1; completionLineStartIdx < completionLines.length; completionLineStartIdx++) { + let matched = true; + let docSkippedEmptyLineCount = 0; + let completionSkippedEmptyLineCount = 0; + for ( + let offset = 0; + offset + completionLineStartIdx + completionSkippedEmptyLineCount < completionLines.length; + offset++ + ) { + let docLine: string | undefined; + while (true) { + const docLineIdx = position.line + 1 + offset + docSkippedEmptyLineCount; + docLine = docLineIdx >= doc.getLineCount() ? undefined : doc.getLineText(docLineIdx); + if (docLine !== undefined && docLine.trim() === '') { + // Skip empty lines in the document and loop + docSkippedEmptyLineCount++; + } else { + break; + } + } + + let completionLineIdx: number | undefined; + let completionLine: string | undefined; + while (true) { + completionLineIdx = completionLineStartIdx + offset + completionSkippedEmptyLineCount; + completionLine = + completionLineIdx >= completionLines.length ? undefined : completionLines[completionLineIdx]; + if (completionLine !== undefined && completionLine.trim() === '') { + // Skip empty lines in the completion and loop + completionSkippedEmptyLineCount++; + } else { + break; + } + } + + const isLastCompletionLine = completionLineIdx === completionLines.length - 1; + if ( + !completionLine || + !( + docLine && + (isLastCompletionLine + ? // For the last line, accept any line that starts with the completion line and vice versa. + // This allows for brackets, braces, parentheses, quotes, identifiers like "end" and "fi", + // heredocs, etc. + docLine.startsWith(completionLine) || completionLine.startsWith(docLine) + : // For other lines, strictly require the block close token, and nothing else + docLine === completionLine && completionLine.trim() === blockCloseToken) + ) + ) { + matched = false; + break; + } + } + if (matched) { + const completionWithoutClosingBracketLines = completionLines + .slice(0, completionLineStartIdx) + .join(completionLinesInfo.newLineCharacter); + return completionWithoutClosingBracketLines; + } + } + + return completion; +} + +function splitByNewLine(text: string): { lines: string[]; newLineCharacter: string } { + const newLineCharacter = text.includes('\r\n') ? '\r\n' : '\n'; + return { + lines: text.split(newLineCharacter), + newLineCharacter, + }; +} + +function matchesNextLine( + document: TextDocumentContents, + position: IPosition, + text: string, + shouldTrim: boolean +): boolean { + let nextLine = ''; + let lineNo: number = position.line + 1; + const compareText = shouldTrim ? text.trim() : text; + while (nextLine === '' && lineNo < document.lineCount) { + nextLine = document.lineAt(lineNo).text; + if (shouldTrim) { + nextLine = nextLine.trim(); + } + if (nextLine === compareText) { + return true; + } + lineNo++; + } + return false; +} + +/** + * Post-processed a completion choice in the context of the document where the choice is offered. + */ +export function postProcessChoiceInContext( + accessor: ServicesAccessor, + document: TextDocumentContents, + position: IPosition, + choice: APIChoice, + isMoreMultiline: boolean, + logger: Logger +): APIChoice | undefined { + const logTarget = accessor.get(ICompletionsLogTargetService); + if (isRepetitive(choice.tokens)) { + const telemetryData = TelemetryData.createAndMarkAsIssued(); + telemetryData.extendWithRequestId(choice.requestId); + telemetry(accessor, 'repetition.detected', telemetryData, TelemetryStore.Enhanced); + // FIXME: trim request at start of repetitive block? for now we just skip + logger.info(logTarget, 'Filtered out repetitive solution'); + return undefined; + } + + const postProcessedChoice = { ...choice }; + + // Avoid single-line completions that duplicate the next line (#993) + if (matchesNextLine(document, position, postProcessedChoice.completionText, !isMoreMultiline)) { + const baseTelemetryData = TelemetryData.createAndMarkAsIssued(); + baseTelemetryData.extendWithRequestId(choice.requestId); + telemetry(accessor, 'completion.alreadyInDocument', baseTelemetryData); + telemetry( + accessor, + 'completion.alreadyInDocument', + baseTelemetryData.extendedBy({ + completionTextJson: JSON.stringify(postProcessedChoice.completionText), + }), + TelemetryStore.Enhanced + ); + logger.info(logTarget, 'Filtered out solution matching next line'); + return undefined; + } + + // Avoid double-closing blocks (#272) + postProcessedChoice.completionText = maybeSnipCompletion( + accessor, + document, + position, + postProcessedChoice.completionText + ); + + return postProcessedChoice.completionText ? postProcessedChoice : undefined; +} + +export function checkSuffix(document: TextDocumentContents, position: IPosition, choice: APIChoice): number { + const currentLine = document.lineAt(position.line); + const restOfLine = currentLine.text.substring(position.character); + if (restOfLine.length > 0) { + if (choice.completionText.indexOf(restOfLine) !== -1) { + //If current suggestion contains rest of the line as substring + //then we will include it in our suggestion range + return restOfLine.length; + } else { + let lastIndex = -1; + let suffixLength = 0; + for (const c of restOfLine) { + const idx = choice.completionText.indexOf(c, lastIndex + 1); + if (idx > lastIndex) { + suffixLength++; + lastIndex = idx; + } else { + break; + } + } + return suffixLength; + } + } + return 0; +} diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/test/anomalyDetection.test.ts b/completions-sample-code/vscode-node/lib/src/suggestions/test/anomalyDetection.test.ts new file mode 100644 index 0000000..ac9d063 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/test/anomalyDetection.test.ts @@ -0,0 +1,109 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { isRepetitive } from '../anomalyDetection'; + +suite('Anomaly Repetition Tests', function () { + test('recognizes sequence consisting of single repeated token', function () { + const tokens = 'Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar'.split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be recognized.'); + }); + + test('does nothing on a too short sequence of single repeated token', function () { + const tokens = 'Bar Bar Bar Bar Bar Bar Bar Bar Bar'.split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, false, 'Repetition should not be recognized.'); + }); + + test('recognizes single repeated token in proper suffix', function () { + const tokens = 'Baz Baz Baz Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar Bar'.split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be recognized.'); + }); + + test('recognizes repeated pattern', function () { + const tokens = ( + 'Bar Far Car Bar Far Car Bar Far Car Bar Far Car Bar Far Car Bar Far Car ' + + 'Bar Far Car Bar Far Car Bar Far Car Bar Far Car Bar Far Car Bar Far Car' + ).split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be recognized.'); + }); + + test('does nothing on a too short repeated pattern', function () { + const tokens = ( + 'Bar Far Car Bar Far Car Bar Far Car Bar Far Car Bar Far Car Bar Far Car ' + + 'Bar Far Car Bar Far Car Bar Far Car' + ).split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, false, 'Repetition should not be recognized.'); + }); + + test('does nothing in absence of a pattern', function () { + const tokens = ( + '12 1 23 43 ac er gf gf 12 er gd 34 dg 35 ;o lo 34 xc ' + + '4t ggf gf 46 l7 dg qs 5y ku df 34 gr gr gr df er gr gr' + ).split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, false, 'No repetition should be claimed.'); + }); + + test('does nothing on too long a pattern', function () { + const tokens = '12 1 23 43 ac er gf gf 12 er gd '.repeat(4).split(' '); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, false, 'No repetition should be claimed.'); + }); + + test('recognizes short real world example', function () { + const tokens = [ + 'C', + ' LIM', + 'IT', + ' 1', + ')', + '\n', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + '\t', + ]; + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be found.'); + }); + + test('recognizes long real world example', function () { + const tokens = + 'Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the website. Try to use the keyboard to navigate the'.split( + ' ' + ); + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be found.'); + }); + + test('recognizes repetitions with some prefix', function () { + const tokens = ['prefix', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo']; + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be found.'); + }); + + test('recognizes repetitions that differ only in whitespace tokens, with some prefix', function () { + const tokens = ['prefix', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', 'foo', ' ', 'foo']; + const repetitive = isRepetitive(tokens); + assert.strictEqual(repetitive, true, 'Repetition should be found.'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/test/editDistance.test.ts b/completions-sample-code/vscode-node/lib/src/suggestions/test/editDistance.test.ts new file mode 100644 index 0000000..36fe304 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/test/editDistance.test.ts @@ -0,0 +1,335 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as ed from '../editDistance'; + +suite('Edit-Distance Test Suite', function () { + function test_alignment(haystack: string, needle: string, expected_dist: number, expected_match: string) { + const alignment = ed.editDistance(haystack, needle); + const alignedStr = haystack.substring(alignment.startOffset, alignment.endOffset); + assert.strictEqual(alignment.distance, expected_dist); + assert.strictEqual(alignedStr, expected_match); + return alignment; + } + + test('perfect match', function () { + test_alignment('XXXXabcYYYY', 'abc', 0, 'abc'); + }); + + test('first perfect match is used', function () { + const alignment = test_alignment('XXXXabcYYYYabcZZZZ', 'abc', 0, 'abc'); + assert.strictEqual(alignment.startOffset, 4); + }); + + test('first equally-good match is used', function () { + test_alignment('XXXXaScdXXXXabdXXXXabYcdXXXX', 'abcd', 1, 'aScd'); + test_alignment('XXXXabdXXXXabYcdXXXXaScdXXXX', 'abcd', 1, 'abd'); + test_alignment('XXXXabYcdXXXXaScdXXXXabdXXXX', 'abcd', 1, 'abYcd'); + }); + + test('complete non-match', function () { + const alignment = test_alignment('XXXX', 'YYY', 3, ''); + assert.strictEqual(alignment.startOffset, 0); + assert.strictEqual(alignment.endOffset, 0); + }); + + test('almost non-match beginning', function () { + test_alignment('aXXX', 'YYa', 2, 'a'); + }); + + test('almost non-match end', function () { + test_alignment('XXa', 'aYY', 2, 'a'); + }); + + test('prefer substitution over equals + insertion', function () { + // Same distance in edit operations. This is a convention + test_alignment('aXbc', 'abc', 1, 'Xbc'); + // Alternative: match "aXbc" as equals on 'a' + insertion of 'X' + }); + + test('prefer deletion over substitution', function () { + // Same distance in edit operations. This is a convention + test_alignment('abcS', 'abcd', 1, 'abc'); + // Alternative: match "abcS" as subst 'd' by 'S' + }); + + test('deletions', function () { + test_alignment('XXXabXcdXefXghXXX', 'abcdefgh', 3, 'abXcdXefXgh'); + }); + + test('substitutions', function () { + test_alignment('abSdeSg', 'abcdefg', 2, 'abSdeSg'); + }); + + test('deletions and substitutions', function () { + test_alignment('XXXabXSdeSXghXXX', 'abcdefgh', 4, 'abXSdeSXgh'); + }); + + test('insertions from start of needle', function () { + test_alignment('deXfgSiXXX', 'abcdefghi', 5, 'deXfgSi'); + }); + + test('insertions from end of needle', function () { + test_alignment('XXXXabXcdSf', 'abcdefgh', 4, 'abXcdSf'); + }); + + test('insertions from the middle of needle', function () { + test_alignment('XXXXabXcfXghXXXXX', 'abcdefgh', 4, 'abXcfXgh'); + }); + + test('single char needle not matched', function () { + test_alignment('XXX', 'a', 1, ''); + }); + + test('single char needle match', function () { + test_alignment('ab', 'a', 0, 'a'); + }); + + test('empty haystack', function () { + test_alignment('', 'abc', 3, ''); + }); + + test('empty needle', function () { + test_alignment('abc', '', 0, ''); + }); + + test('empty needle and haystack', function () { + test_alignment('', '', 0, ''); + }); +}); + +suite('Lexical Analyzer for Edit-Distance Test Suite', function () { + function test_lexing(s: string, expected_lexemes: string[]) { + const [lexemes, d] = ed.lexicalAnalyzer(s, ed.emptyLexDictionary(), ed.lexGeneratorWords, lexeme => true); + const lookup = ed.reverseLexDictionary(d); + assert.deepStrictEqual( + lexemes.map(([lid]) => lookup[lid]), + expected_lexemes + ); + } + + test('lex some alphanumeric words with underscores', function () { + test_lexing('abc as22_b 12abc aAzZu', ['abc', ' ', 'as22_b', ' ', '12abc', ' ', 'aAzZu']); + }); + + test('lex split at symbols', function () { + test_lexing('a:a-a;a+a?a{a}a(a)$a#a@a!a=a\\a\'a"a&a^', [ + 'a', + ':', + 'a', + '-', + 'a', + ';', + 'a', + '+', + 'a', + '?', + 'a', + '{', + 'a', + '}', + 'a', + '(', + 'a', + ')', + '$', + 'a', + '#', + 'a', + '@', + 'a', + '!', + 'a', + '=', + 'a', + '\\', + 'a', + `'`, + 'a', + '"', + 'a', + '&', + 'a', + '^', + ]); + }); + + test('lex spaces and other whitespace', function () { + test_lexing(' a a a a \n a \t a \r a\n\n', [ + ' ', + 'a', + ' ', + 'a', + ' ', + 'a', + ' ', + 'a', + ' ', + '\n', + ' ', + 'a', + ' ', + '\t', + ' ', + 'a', + ' ', + '\r', + ' ', + 'a', + '\n', + '\n', + ]); + }); + + test('lex common double-character symbols take up two lexemes', function () { + test_lexing('== => -> ::', ['=', '=', ' ', '=', '>', ' ', '-', '>', ' ', ':', ':']); + }); + + test('lex astral plane characters', function () { + test_lexing(' a๐Ÿคช๐Ÿคซa๐Ÿคฌ1๐Ÿคญ_๐Ÿคฎ', [' ', 'a', '๐Ÿคช', '๐Ÿคซ', 'a', '๐Ÿคฌ', '1', '๐Ÿคญ', '_', '๐Ÿคฎ']); + }); + + test('lex alternative alphabets form words', function () { + // Write some greek letters + test_lexing( + 'a\u03B1\u03B2\u03B3\u03B4\u03B5\u03B6\u03B7\u03B8\u03B9\u03BA\u03BB\u03BC\u03BD\u03BE\u03BF\u03C0\u03C1\u03C2\u03C3\u03C4\u03C5', + [ + 'a\u03B1\u03B2\u03B3\u03B4\u03B5\u03B6\u03B7\u03B8\u03B9\u03BA\u03BB\u03BC\u03BD\u03BE\u03BF\u03C0\u03C1\u03C2\u03C3\u03C4\u03C5', + ] + ); + }); + + function test_lex_alignment(haystack: string, needle: string, expected_lex_dist: number, expected_match: string) { + const alignment = ed.lexEditDistance(haystack, needle); + const alignedStr = haystack.substring(alignment.startOffset, alignment.endOffset); + assert.strictEqual(expected_lex_dist, alignment.lexDistance); + assert.strictEqual(expected_match, alignedStr); + return alignment; + } + + test('lex-edit-dist perfect match', function () { + test_lex_alignment('XX XX a b c\nd YY YY', 'a b c\nd', 0, 'a b c\nd'); + }); + + test('lex-edit-dist ignores single spaces', function () { + test_lex_alignment('XX XX ( ) { YY YY', '(){', 0, '( ) {'); + }); + + test('lex-edit-dist counts multiple spaces and newlines', function () { + test_lex_alignment('XX XX def fun ( )\n {z} YY YY', 'def fun (){z}', 3, 'def fun ( )\n {z}'); + }); + + test('lex-edit-dist long words small distance', function () { + test_lex_alignment( + 'a bee is a tee in deed', + 'a hippopotamus is a pachyderm in deed', + 2, + 'a bee is a tee in deed' + ); + }); + + test('lex-edit-dist first needle lexeme match postfix of lexeme in haystack', function () { + test_lex_alignment('AKingdomForAHorse he did cry', 'Horse he did', 0, 'AKingdomForAHorse he did'); + }); + + test('lex-edit-dist last needle lexeme match prefix of lexeme in haystack', function () { + test_lex_alignment( + 'uncomfortable with promptOrExplode', + 'comfortable with prompt', + 0, + 'uncomfortable with promptOrExplode' + ); + }); + + test('lex-edit-dist needle single lexeme match postfix', function () { + test_lex_alignment('xx aabb cc dd', 'abb', 0, 'aabb'); + }); + + test('lex-edit-dist needle single lexeme match prefix', function () { + test_lex_alignment('xx aabb cc dd', 'aab', 0, 'aabb'); + }); + + test('lex-edit-dist haystack single lexeme match postfix', function () { + test_lex_alignment('aabb', 'abb cc', 1, 'aabb'); + }); + + test('lex-edit-dist haystack single lexeme match prefix', function () { + test_lex_alignment('aabb', 'cc aab', 1, 'aabb'); + }); + + // The following tests are equivalent to those for character-based + // edit-distance, but all characters have been replaced by multi-character + // tokens. This is more test of offset-adjustment rather than the + // edit-distance algorithm itself. + + test('lexed almost non-match beginning', function () { + test_lex_alignment('aa XX XX XX ', 'YY YY aa', 2, 'aa'); + }); + + test('lexed almost non-match end', function () { + test_lex_alignment('XX XX aa', 'aa YY YY ', 2, 'aa'); + }); + + test('lexed prefer substitution over equals + insertion', function () { + test_lex_alignment('aa XX bb cc ', 'aa bb cc ', 1, 'XX bb cc'); + }); + + test('lexed prefer deletion over substitution', function () { + // Same distance in edit operations. This is a convention + test_lex_alignment('aa bb cc SS ', 'aa bb cc dd', 1, 'aa bb cc'); + }); + + test('lexed deletions', function () { + test_lex_alignment( + 'XX XX XX aa bb XX cc dd XX ee ff XX gg hh XX XX XX ', + 'aa bb cc dd ee ff gg hh', + 3, + 'aa bb XX cc dd XX ee ff XX gg hh' + ); + }); + + test('lexed substitutions', function () { + test_lex_alignment('aa bb SS dd ee SS gg', 'aa bb cc dd ee ff gg', 2, 'aa bb SS dd ee SS gg'); + }); + + test('lexed deletions and substitutions', function () { + test_lex_alignment( + 'XX XX XX aa bb XX SS dd ee SS XX gg hh XX XX XX ', + 'aa bb cc dd ee ff gg hh', + 4, + 'aa bb XX SS dd ee SS XX gg hh' + ); + }); + + test('lexed insertions from start of needle', function () { + test_lex_alignment('dd ee XX ff gg SS ii XX XX XX ', 'aa bb cc dd ee ff gg hh ii', 5, 'dd ee XX ff gg SS ii'); + }); + + test('lexed insertions from end of needle', function () { + test_lex_alignment('XX XX XX XX aa bb XX cc dd SS ff', 'aa bb cc dd ee ff gg hh', 4, 'aa bb XX cc dd SS ff'); + }); + + test('lexed insertions from the middle of needle', function () { + test_lex_alignment( + 'XX XX XX XX aa bb XX cc ff XX gg hh XX XX XX XX XX ', + 'aa bb cc dd ee ff gg hh', + 4, + 'aa bb XX cc ff XX gg hh' + ); + }); + + test('lexed empty haystack', function () { + test_lex_alignment('', 'aa bb cc', 3, ''); + }); + + test('lexed empty needle', function () { + test_lex_alignment('aa bb cc', '', 0, ''); + }); + + test('lexed empty needle and haystack', function () { + test_lex_alignment('', '', 0, ''); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/test/partialSuggestions.test.ts b/completions-sample-code/vscode-node/lib/src/suggestions/test/partialSuggestions.test.ts new file mode 100644 index 0000000..4caac60 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/test/partialSuggestions.test.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { + SuggestionStatus, + computeCompCharLen, + computeCompletionText, + countLines, +} from '../partialSuggestions'; + +suite('partial acceptance utilities', () => { + test('returns the length of the completion text when compType is full', () => { + const completionText = 'Hello, World!'; + const suggestionStatus: SuggestionStatus = { + compType: 'full', + acceptedLength: completionText.length, + acceptedLines: 0, + }; + + const result = computeCompCharLen(suggestionStatus, completionText); + + assert.strictEqual(result, completionText.length); + }); + + test('returns the acceptedLength when compType is partial', () => { + const acceptedLength = 5; + const suggestionStatus: SuggestionStatus = { compType: 'partial', acceptedLength, acceptedLines: 0 }; + + const result = computeCompCharLen(suggestionStatus, 'Hello, World!'); + + assert.strictEqual(result, acceptedLength); + }); + + test('returns the full completion text when compType is full', () => { + const completionText = 'Hello, World!'; + const suggestionStatus: SuggestionStatus = { + compType: 'full', + acceptedLength: completionText.length, + acceptedLines: 0, + }; + + const result = computeCompletionText(completionText, suggestionStatus); + + assert.strictEqual(result, completionText); + }); + + test('returns the substring of the completion text when compType is partial', () => { + const acceptedLength = 5; + const completionText = 'Hello, World!'; + const suggestionStatus: SuggestionStatus = { compType: 'partial', acceptedLength, acceptedLines: 0 }; + + const result = computeCompletionText(completionText, suggestionStatus); + + assert.strictEqual(result, 'Hello'); + }); +}); + +suite('countLines function', () => { + test('returns 0 for empty string', () => { + const result = countLines(''); + assert.strictEqual(result, 0); + }); + + test('returns 1 for single line without newline', () => { + const result = countLines('single line text'); + assert.strictEqual(result, 1); + }); + + test('handles Unix newlines (\\n)', () => { + const text = 'line1\nline2\nline3'; + const result = countLines(text); + + assert.strictEqual(result, 3); + }); + + test('handles Windows newlines (\\r\\n)', () => { + const text = 'line1\r\nline2\r\nline3'; + const result = countLines(text); + + assert.strictEqual(result, 3); + }); + + test('ignores old Mac newlines (\\r)', () => { + const text = 'line1\rline2'; + const result = countLines(text); + + assert.strictEqual(result, 1); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/suggestions/test/suggestions.test.ts b/completions-sample-code/vscode-node/lib/src/suggestions/test/suggestions.test.ts new file mode 100644 index 0000000..de2d4ec --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/suggestions/test/suggestions.test.ts @@ -0,0 +1,247 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { Position } from 'vscode'; +import { APIChoice } from '../../openai/openai'; +import { createTextDocument } from '../../test/textDocument'; +import { ILines, checkSuffix, maybeSnipCompletionImpl } from '../suggestions'; + +suite('checkSuffix', function () { + function assertSuffix(completionText: string, lineSuffix: string, expected: number) { + const doc = createTextDocument('file:///foo', 'typescript', 1, lineSuffix); + const processed = checkSuffix(doc, { line: 0, character: 0 }, <APIChoice>{ + completionText, + }); + assert.strictEqual(processed, expected); + } + + test('consecutive', function () { + assertSuffix('foo({});', '});', 3); + }); + + test('nonconsecutive', function () { + assertSuffix('foo("bar", {});', '");', 3); + }); +}); + +suite('Test maybeSnipCompletionImpl', function () { + test('Test maybeSnipCompletionImpl single closing bracket', function () { + const lines = new StaticLines(` +class LicenseStore { + public readonly filePath: string; + public readonly fullLicenseText: { [key: string]: string[] }; + + constructor(filePath: string, fullLicenseText: @ + } +} + `); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines, + lines.getPositionOfAt()!, + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + }`, + '}' + ), + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText;` + ); + }); + + test('Test maybeSnipCompletionImpl double closing bracket', function () { + const lines = new StaticLines(` +class LicenseStore { + public readonly filePath: string; + public readonly fullLicenseText: { [key: string]: string[] }; + + constructor(filePath: string, fullLicenseText: @ + } +} + `); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines, + lines.getPositionOfAt()!, + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + } +}`, + '}' + ), + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText;` + ); + }); + + test('Test maybeSnipCompletionImpl single closing bracket with semicolon', function () { + const lines = new StaticLines(` +class LicenseStore { + public readonly filePath: string; + public readonly fullLicenseText: { [key: string]: string[] }; + + constructor(filePath: string, fullLicenseText: @ + } +} + `); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines, + lines.getPositionOfAt()!, + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + };`, + '}' + ), + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText;` + ); + }); + + test('Test maybeSnipCompletionImpl: Only last line can just be a prefix of the model line', function () { + const lines = new StaticLines(` +class LicenseStore { + public readonly filePath: string; + public readonly fullLicenseText: { [key: string]: string[] }; + + constructor(filePath: string, fullLicenseText: @ + }1 +}2 + `); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines, + lines.getPositionOfAt()!, + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + } +}`, + '}' + ), + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + } +}` + ); + + // Not restricted to the block close token + const lines2 = new StaticLines(` +const list [ + @ +]; + `); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines2, + lines2.getPositionOfAt()!, + `'one', + 'two', + 'three' +]`, + '}' + ), + `'one', + 'two', + 'three'` + ); + }); + + test('Test maybeSnipCompletionImpl: The last line can just be a prefix of the model line', function () { + const lines = new StaticLines(` +class LicenseStore { + public readonly filePath: string; + public readonly fullLicenseText: { [key: string]: string[] }; + + constructor(filePath: string, fullLicenseText: @ + } +}2 + `); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines, + lines.getPositionOfAt()!, + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + } +}`, + '}' + ), + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText;` + ); + }); + + test('Test maybeSnipCompletionImpl: Empty Lines In Completion', function () { + const lines = new StaticLines(` +class LicenseStore { + public readonly filePath: string; + public readonly fullLicenseText: { [key: string]: string[] }; + + constructor(filePath: string, fullLicenseText: @ + + } + +}`); + + assert.deepStrictEqual( + maybeSnipCompletionImpl( + lines, + lines.getPositionOfAt()!, + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText; + } + + +}`, + '}' + ), + `any) { + this.filePath = filePath; + this.fullLicenseText = fullLicenseText;` + ); + }); +}); + +class StaticLines implements ILines { + private readonly lines: string[]; + constructor(text: string) { + this.lines = text.split(/\r\n|\n/g); + } + + getLineText(lineIdx: number): string { + return this.lines[lineIdx]; + } + + getLineCount(): number { + return this.lines.length; + } + + getPositionOfAt(): Position | undefined { + for (let i = 0; i < this.lines.length; i++) { + const idx = this.lines[i].indexOf('@'); + if (idx !== -1) { + return new Position(i, idx); + } + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/telemetry.ts b/completions-sample-code/vscode-node/lib/src/telemetry.ts new file mode 100644 index 0000000..c614020 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/telemetry.ts @@ -0,0 +1,680 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { IEnvService } from '../../../../../platform/env/common/envService'; +import { RequestId } from '../../../../../platform/networking/common/fetch'; +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { generateUuid } from '../../../../../util/vs/base/common/uuid'; +import { IInstantiationService, ServicesAccessor } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTelemetryService } from '../../bridge/src/completionsTelemetryServiceBridge'; +import { + BuildInfo, + dumpForTelemetry, + formatNameAndVersion, ICompletionsEditorAndPluginInfo +} from './config'; +import { ExpConfig } from './experiments/expConfig'; +import { ICompletionsFeaturesService } from './experiments/featuresService'; +import { FilterSettings } from './experiments/filters'; +import { ExpServiceTelemetryNames } from './experiments/telemetryNames'; +import { APIJsonData } from './openai/openai'; +import { Prompt } from './prompt/prompt'; +import { ICompletionsTelemetryUserConfigService } from './telemetry/userConfig'; +import { ICompletionsPromiseQueueService } from './util/promiseQueue'; + +export enum TelemetryStore { + Standard, + Enhanced, +} + +export namespace TelemetryStore { + export function isEnhanced(store: TelemetryStore): boolean { + return store === TelemetryStore.Enhanced; + } +} + +function isEnhanced(store: TelemetryStore): boolean { + return store === TelemetryStore.Enhanced; +} + +const ftTelemetryEvents = [ + 'engine.prompt', + 'engine.completion', + 'ghostText.capturedAfterAccepted', + 'ghostText.capturedAfterRejected', +]; + +const MAX_PROPERTY_LENGTH = 8192; +// The largest context size we have today is 168k which can fit in 21 properties of 8k each. +const MAX_CONCATENATED_PROPERTIES = 21; + +export type TelemetryProperties = { [key: string]: string }; +export type TelemetryMeasurements = { [key: string]: number }; + + +/** + * A class holding the data we want send to telemetry, + * {@link TelemetryData.properties} containing the strings + * and {@link TelemetryData.measurements} containing the numbers. + * + * Additionally, this keeps tracks of timestamps {@link TelemetryData.created} and {@link TelemetryData.displayed} + * that can be used to track when this object was created or when information + * contained in this object was surfaced to the user. + * + * This is meant be used as an argument to + * {@link telemetry}, {@link telemetryError}, or {@link telemetryException}. + */ +export class TelemetryData { + properties: TelemetryProperties; + measurements: TelemetryMeasurements; + issuedTime: number; + displayedTime?: number; + + private static keysExemptedFromSanitization: string[] = [ + ExpServiceTelemetryNames.featuresTelemetryPropertyName, + ]; + + protected constructor(properties: TelemetryProperties, measurements: TelemetryMeasurements, issuedTime: number) { + this.properties = properties; + this.measurements = measurements; + this.issuedTime = issuedTime; + } + + static createAndMarkAsIssued( + properties?: TelemetryProperties, + measurements?: TelemetryMeasurements + ): TelemetryData { + return new TelemetryData(properties || {}, measurements || {}, now()); + } + + /** + * @param properties new properties, which will overwrite old ones in case of a clash + * @param measurements new measurements, which will overwrite old ones in case of a clash + * @returns a TelemetryData object whose contents extend (copies of) the current one's and whose creation date is not updated + */ + extendedBy(properties?: TelemetryProperties, measurements?: TelemetryMeasurements): TelemetryData { + const newProperties = { ...this.properties, ...properties }; + const newMeasurements = { ...this.measurements, ...measurements }; + const newData = new TelemetryData(newProperties, newMeasurements, this.issuedTime); + newData.displayedTime = this.displayedTime; + + return newData; + } + + /** + * registers current time as the point where this was displayed + * (no-op if a display time is already registered) + */ + markAsDisplayed(): void { + if (this.displayedTime === undefined) { + this.displayedTime = now(); + } + } + + /** This function is a fallback - if we are a TelemetryData object instead of a TelemetryWithExp, + * we don't actually know our real ExP assignment list. Historically, all telemetry has been emitted + * with a 'partial' list of assignments that are gathered using a blank set of filters, and there may + * be downstream telemetry users depending on this partial list. + * However, this partial list likely disagrees with the true, complete list that TelemetryWithExp + * can emit, so there is the possibility of inconsistent telemetry (different events from the same user/context + * will have different experimental assignments). + * All telemetry events that impact scorecards (namely ghostText) MUST use TelemetryWithExp, but + * this fallback is a bandaid for other events that don't impact scorecards. + * Downstream users SHOULD NOT depend on the partial list, and this fallback should eventually be removed + * in favor of properly plumbing a TelemetryWithExp object through in the cases where the + * assignment list is necessary. + */ + async extendWithExpTelemetry(accessor: ServicesAccessor): Promise<void> { + const { filters, exp } = await accessor.get(ICompletionsFeaturesService).getFallbackExpAndFilters(); + exp.addToTelemetry(this); + filters.addToTelemetry(this); + } + + extendWithEditorAgnosticFields(accessor: ServicesAccessor): void { + const envService = accessor.get(IEnvService); + const editorAndPluginInfo = accessor.get(ICompletionsEditorAndPluginInfo); + + this.properties['editor_version'] = formatNameAndVersion(editorAndPluginInfo.getEditorInfo()); + this.properties['editor_plugin_version'] = formatNameAndVersion( + editorAndPluginInfo.getEditorPluginInfo() + ); + this.properties['client_machineid'] = envService.machineId; + this.properties['client_sessionid'] = envService.sessionId; + this.properties['copilot_version'] = `copilot/${BuildInfo.getVersion()}`; + if (typeof process !== 'undefined') { + this.properties['runtime_version'] = `node/${process.versions.node}`; + } + + this.properties['common_extname'] = editorAndPluginInfo.getEditorPluginInfo().name; + this.properties['common_extversion'] = editorAndPluginInfo.getEditorPluginInfo().version; + this.properties['common_vscodeversion'] = formatNameAndVersion(editorAndPluginInfo.getEditorInfo()); + } + + /** + * Iterate config keys defined in the package.json, lookup current config + * value and return as telemetry property. Property name in dotted notation + * and value is a json string. + * e.g. { 'copilot.autocompletion.count': 3 } + */ + extendWithConfigProperties(accessor: ServicesAccessor): void { + const configProperties: { [key: string]: string } = dumpForTelemetry(accessor); + configProperties['copilot.build'] = BuildInfo.getBuild(); + configProperties['copilot.buildType'] = BuildInfo.getBuildType(); + + // By being the second argument, configProperties will always override + this.properties = { ...this.properties, ...configProperties }; + } + + extendWithRequestId(requestId: RequestId): void { + const requestProperties = { + headerRequestId: requestId.headerRequestId, + serverExperiments: requestId.serverExperiments, + deploymentId: requestId.deploymentId, + }; + this.properties = { ...this.properties, ...requestProperties }; + } + + private static keysToRemoveFromStandardTelemetry: string[] = [ + 'gitRepoHost', + 'gitRepoName', + 'gitRepoOwner', + 'gitRepoUrl', + 'gitRepoPath', + 'repo', + 'request_option_nwo', + 'userKind', + ]; + + /** + * Remove the known properties relating to repository information from the telemetry data if necessary + */ + static maybeRemoveRepoInfoFromProperties( + store: TelemetryStore, + map: { [key: string]: string } + ): { [key: string]: string } { + if (isEnhanced(store)) { + // We want to keep including these properties in enhanced telemetry. + return map; + } + // deliberately written in the same style as `sanitizeKeys` to minimise risk + const returnValue: { [key: string]: string } = {}; + for (const key in map) { + if (!TelemetryData.keysToRemoveFromStandardTelemetry.includes(key)) { + returnValue[key] = map[key]; + } + } + return returnValue; + } + + sanitizeKeys(): void { + this.properties = TelemetryData.sanitizeKeys(this.properties); + this.measurements = TelemetryData.sanitizeKeys(this.measurements); + // Not just keys anymore, also values + for (const key in this.measurements) { + if (isNaN(this.measurements[key])) { + delete this.measurements[key]; + } + } + } + + multiplexProperties(): void { + this.properties = TelemetryData.multiplexProperties(this.properties); + } + + static sanitizeKeys<V>(map?: { [key: string]: V }): { [key: string]: V } { + // We need all keys to not have dots in them for telemetry to function + map = map || {}; + const returnValue: { [key: string]: V } = {}; + // Iterate over all keys in the map and replace dots with underscores + for (const key in map) { + const newKey = TelemetryData.keysExemptedFromSanitization.includes(key) ? key : key.replace(/\./g, '_'); + returnValue[newKey] = map[key]; + } + return returnValue; + } + + static multiplexProperties(properties: TelemetryProperties): TelemetryProperties { + const newProperties = { ...properties }; + for (const key in properties) { + const value = properties[key]; + // Test the length of value + let remainingValueCharactersLength = value?.length ?? 0; + if (remainingValueCharactersLength > MAX_PROPERTY_LENGTH) { + let lastStartIndex = 0; + let newPropertiesCount = 0; + while (remainingValueCharactersLength > 0 && newPropertiesCount < MAX_CONCATENATED_PROPERTIES) { + newPropertiesCount += 1; + let propertyName = key; + if (newPropertiesCount > 1) { + propertyName = key + '_' + (newPropertiesCount < 10 ? '0' : '') + newPropertiesCount; + } + let offsetIndex = lastStartIndex + MAX_PROPERTY_LENGTH; + if (remainingValueCharactersLength < MAX_PROPERTY_LENGTH) { + offsetIndex = lastStartIndex + remainingValueCharactersLength; + } + newProperties[propertyName] = value.slice(lastStartIndex, offsetIndex); + remainingValueCharactersLength -= MAX_PROPERTY_LENGTH; + lastStartIndex += MAX_PROPERTY_LENGTH; + } + } + } + return newProperties; + } + + updateMeasurements(now: number): void { + const timeSinceIssued = now - this.issuedTime; + this.measurements.timeSinceIssuedMs = timeSinceIssued; + + if (this.displayedTime !== undefined) { + const timeSinceDisplayed = now - this.displayedTime; + this.measurements.timeSinceDisplayedMs = timeSinceDisplayed; + } + + // Set the current time right before sending the telemetry. + if (this.measurements.current_time === undefined) { + // Because of the way CTS converts the time, we can only get the current time in seconds. + this.measurements.current_time = nowSeconds(now); + } + } + + // Now is passed as an argument to avoid any measurement discrepancies due to + // async operations in the telemetry event. + async makeReadyForSending( + accessor: ServicesAccessor, + store: TelemetryStore, + includeExp: 'IncludeExp' | 'SkipExp', + now: number + ): Promise<void> { + const instantiationService = accessor.get(IInstantiationService); + this.extendWithConfigProperties(accessor); + this.extendWithEditorAgnosticFields(accessor); + this.sanitizeKeys(); + this.multiplexProperties(); + // the `includeExp` parameter is so we don't get into an infinite loop sending telemetry about + // ExP itself. + if (includeExp === 'IncludeExp') { + // we actually want to do this step _after_ sanitizing the keys, because the keys may be unsanitary (and still required) + await this.extendWithExpTelemetry(accessor); + } + this.updateMeasurements(now); + Object.assign(this.properties, instantiationService.invokeFunction(createRequiredProperties)); + } +} + +/** + * A TelemetryData object that also contains the filters and ExP config that are applicable to current request context. + * Telemetry which is used to generate scorecards *must* use this class over the bare TelemetryData class in order + * to guarantee that the events are attached to the correct scorecard. Known events that fall into this category are: + * - `ghostText.issued` + * - `ghostText.shown` + * - `ghostText.accepted` + * - `ghostText.performance` + * + * It is highly recommended to use this class for most other telemetry events as well, to ensure that the events can be + * tied correctly to active experiments in post-hoc analyses. + * + * This object should only be created directly by the `updateExPValuesAndAssignments` function of `experiments/features.ts`, + * unless testing. + * + * This class should not be used for telemetry that does not take place in the context of a "completion request". + */ +export class TelemetryWithExp extends TelemetryData { + filtersAndExp: { filters: FilterSettings; exp: ExpConfig }; + + constructor( + properties: TelemetryProperties, + measurements: TelemetryMeasurements, + issuedTime: number, + filtersAndExp: { filters: FilterSettings; exp: ExpConfig } + ) { + super(properties, measurements, issuedTime); + this.filtersAndExp = filtersAndExp; + } + + override extendedBy(properties?: TelemetryProperties, measurements?: TelemetryMeasurements): TelemetryWithExp { + const newProperties = { ...this.properties, ...properties }; + const newMeasurements = { ...this.measurements, ...measurements }; + const newData = new TelemetryWithExp(newProperties, newMeasurements, this.issuedTime, this.filtersAndExp); + newData.displayedTime = this.displayedTime; + + return newData; + } + + /** Include the known ExP assignment list into the properties/measurements blocks + * of the telemetry event. + * This method is correct/consistent for TelemetryWithExp, unlike TelemetryData's. + */ + override extendWithExpTelemetry(): Promise<void> { + this.filtersAndExp.exp.addToTelemetry(this); + this.filtersAndExp.filters.addToTelemetry(this); + return Promise.resolve(); + } + + static createEmptyConfigForTesting(): TelemetryWithExp { + return new TelemetryWithExp({}, {}, 0, { + filters: new FilterSettings({}), + exp: ExpConfig.createEmptyConfig(), + }); + } +} + +// Helpers +function sendTelemetryEvent( + completionsTelemetryService: ICompletionsTelemetryService, + store: TelemetryStore, + name: string, + data: { properties: TelemetryProperties; measurements: TelemetryMeasurements } +): void { + const properties = TelemetryData.maybeRemoveRepoInfoFromProperties(store, data.properties); + if (!isEnhanced(store)) { + completionsTelemetryService.sendGHTelemetryEvent( + name, + properties, + data.measurements + ); + } else { + completionsTelemetryService.sendEnhancedGHTelemetryEvent( + name, + properties, + data.measurements + ); + } +} + +function sendTelemetryErrorEvent( + accessor: ServicesAccessor, + store: TelemetryStore, + name: string, + data: { properties: TelemetryProperties; measurements: TelemetryMeasurements } +): void { + const telemetryService = accessor.get(ICompletionsTelemetryService); + const properties = TelemetryData.maybeRemoveRepoInfoFromProperties(store, data.properties); + telemetryService.sendGHTelemetryErrorEvent( + name, + properties, + data.measurements + ); +} + +function sendFTTelemetryEvent( + accessor: ServicesAccessor, + store: TelemetryStore, + name: string, + data: { properties: TelemetryProperties; measurements: TelemetryMeasurements } +): void { + if (!shouldSendFinetuningTelemetry(accessor)) { + return; + } + // const completionsTelemetryService = accessor.get(ICompletionsTelemetryService); + // const properties = TelemetryData.maybeRemoveRepoInfoFromProperties(store, data.properties); + // completionsTelemetryService.sendGHFTTelemetryEvent( + // name, + // properties, + // data.measurements + // ); +} + +/** + * Creates an object containing info about the length of the prompt suitable + * for saving in standard telemetry. + */ +export function telemetrizePromptLength(prompt: Prompt): { [key: string]: number } { + return { + // prefix length + sum of context length + promptCharLen: prompt.prefix.length + (prompt.context?.reduce((sum, c) => sum + c.length, 0) ?? 0), + promptSuffixCharLen: prompt.suffix.length, + }; +} + +export function now(): number { + return performance.now(); +} + +function nowSeconds(now: number): number { + return Math.floor(now / 1000); +} + +function shouldSendEnhanced(accessor: ServicesAccessor): boolean { + return accessor.get(ICompletionsTelemetryUserConfigService).optedIn; +} + +function shouldSendFinetuningTelemetry(accessor: ServicesAccessor): boolean { + return accessor.get(ICompletionsTelemetryUserConfigService).ftFlag !== ''; +} + +export function telemetry(accessor: ServicesAccessor, name: string, telemetryData?: TelemetryData, store?: TelemetryStore) { + return accessor.get(ICompletionsPromiseQueueService).register(_telemetry(accessor, name, now(), telemetryData?.extendedBy(), store)); +} + +async function _telemetry( + accessor: ServicesAccessor, + name: string, + now: number, + telemetryData?: TelemetryData, + store = TelemetryStore.Standard +) { + const completionsTelemetryService = accessor.get(ICompletionsTelemetryService); + const instantiationService = accessor.get(IInstantiationService); + + // if telemetry data isn't given, make a new one to hold at least the config + const definedTelemetryData = telemetryData || TelemetryData.createAndMarkAsIssued({}, {}); + await definedTelemetryData.makeReadyForSending(accessor, store ?? false, 'IncludeExp', now); + if (!isEnhanced(store) || instantiationService.invokeFunction(shouldSendEnhanced)) { + sendTelemetryEvent(completionsTelemetryService, store, name, definedTelemetryData); + } + if (isEnhanced(store) && ftTelemetryEvents.includes(name) && instantiationService.invokeFunction(shouldSendFinetuningTelemetry)) { + instantiationService.invokeFunction(sendFTTelemetryEvent, store, name, definedTelemetryData); + } +} + +export function telemetryExpProblem(accessor: ServicesAccessor, telemetryProperties: { reason: string }) { + const promiseQueueService = accessor.get(ICompletionsPromiseQueueService); + return promiseQueueService.register(_telemetryExpProblem(accessor, telemetryProperties, now())); +} + +async function _telemetryExpProblem(accessor: ServicesAccessor, telemetryProperties: { reason: string }, now: number) { + const completionsTelemetryService = accessor.get(ICompletionsTelemetryService); + const name = 'expProblem'; + const definedTelemetryData = TelemetryData.createAndMarkAsIssued(telemetryProperties, {}); + await definedTelemetryData.makeReadyForSending(accessor, TelemetryStore.Standard, 'SkipExp', now); + sendTelemetryEvent(completionsTelemetryService, TelemetryStore.Standard, name, definedTelemetryData); +} + +/** + * Send a telemetry message as-is, without the usual Copilot-specific processing from + * `createAndMarkAsIssued` / `makeReadyForSending`. + * + * There is also no sanitization or validation currently. When adding new messages + * using this method, make sure to add some tests of the fields, e.g. in `extension/src/ghostTest/telemetry.test.ts`. + */ +export function telemetryRaw( + accessor: ServicesAccessor, + name: string, + props: TelemetryProperties, + measurements: TelemetryMeasurements +) { + const completionsTelemetryService = accessor.get(ICompletionsTelemetryService); + const properties = { ...props, ...createRequiredProperties(accessor) }; + sendTelemetryEvent(completionsTelemetryService, TelemetryStore.Standard, name, { properties, measurements }); +} + +function createRequiredProperties(accessor: ServicesAccessor) { + const editorAndPluginInfo = accessor.get(ICompletionsEditorAndPluginInfo); + const properties: TelemetryProperties = { + unique_id: generateUuid(), // add a unique id to the telemetry event so copilot-foundations can correlate with duplicate events + common_extname: editorAndPluginInfo.getEditorPluginInfo().name, + common_extversion: editorAndPluginInfo.getEditorPluginInfo().version, + common_vscodeversion: formatNameAndVersion(editorAndPluginInfo.getEditorInfo()), + }; + const telemetryConfig = accessor.get(ICompletionsTelemetryUserConfigService); + return { ...telemetryConfig.getProperties(), ...properties }; +} + +export function telemetryException( + telemetryService: ICompletionsTelemetryService, + maybeError: unknown, + transaction: string, +) { + return telemetryService.sendGHTelemetryException(maybeError, transaction || ''); +} + +type TelemetryCatcher = (...args: never[]) => unknown; + +export function telemetryCatch<F extends TelemetryCatcher>( + completionsTelemetryService: ICompletionsTelemetryService, + completionsPromiseQueueService: ICompletionsPromiseQueueService, + fn: F, + transaction: string, +): (...args: Parameters<F>) => void { + const wrapped = async (...args: Parameters<F>) => { + try { + await fn(...args); + } catch (error) { + telemetryException(completionsTelemetryService, error, transaction); + } + }; + return (...args) => completionsPromiseQueueService.register(wrapped(...args)); +} + +export function telemetryError(accessor: ServicesAccessor, name: string, telemetryData?: TelemetryData, store?: TelemetryStore) { + return accessor.get(ICompletionsPromiseQueueService).register(_telemetryError(accessor, name, now(), telemetryData?.extendedBy(), store)); +} + +async function _telemetryError( + accessor: ServicesAccessor, + name: string, + now: number, + telemetryData?: TelemetryData, + store = TelemetryStore.Standard +) { + if (isEnhanced(store) && !shouldSendEnhanced(accessor)) { + return; + } + const instantiationService = accessor.get(IInstantiationService); + const definedTelemetryData = telemetryData || TelemetryData.createAndMarkAsIssued({}, {}); + await definedTelemetryData.makeReadyForSending(accessor, store, 'IncludeExp', now); + instantiationService.invokeFunction(sendTelemetryErrorEvent, store, name, definedTelemetryData); +} + +export function logEngineCompletion( + accessor: ServicesAccessor, + completionText: string, + jsonData: APIJsonData, + requestId: RequestId, + choiceIndex: number +) { + const telemetryData = TelemetryData.createAndMarkAsIssued({ + completionTextJson: JSON.stringify(completionText), + choiceIndex: choiceIndex.toString(), + }); + + if (jsonData.logprobs) { + for (const [key, value] of Object.entries(jsonData.logprobs)) { + telemetryData.properties['logprobs_' + key] = JSON.stringify(value) ?? 'unset'; + } + } + + telemetryData.extendWithRequestId(requestId); + return telemetry(accessor, 'engine.completion', telemetryData, TelemetryStore.Enhanced); +} + +export function logEnginePrompt(accessor: ServicesAccessor, prompt: Prompt, telemetryData: TelemetryData) { + const promptTelemetry: Record<string, string> = { + promptJson: JSON.stringify({ prefix: prompt.prefix, context: prompt.context }), + promptSuffixJson: JSON.stringify(prompt.suffix), + }; + + // Re-add context to stringified request.option.extra if it exists + if (prompt.context) { + const optionExtra = telemetryData.properties['request.option.extra'] + ? (JSON.parse(telemetryData.properties['request.option.extra']) as Record<string, unknown>) + : {}; + optionExtra.context = prompt.context; + promptTelemetry['request.option.extra'] = JSON.stringify(optionExtra); + } + + const telemetryDataWithPrompt = telemetryData.extendedBy(promptTelemetry); + return telemetry(accessor, 'engine.prompt', telemetryDataWithPrompt, TelemetryStore.Enhanced); +} + +// Please don't delete these classes. They are needed for tests. +export abstract class CopilotTelemetryReporter { + abstract sendTelemetryEvent( + eventName: string, + properties?: { + [key: string]: string; + }, + measurements?: { + [key: string]: number; + } + ): void; + abstract sendTelemetryErrorEvent( + eventName: string, + properties?: { + [key: string]: string; + }, + measurements?: { + [key: string]: number; + }, + errorProps?: string[] + ): void; + abstract dispose(): Promise<void>; +} + +export const ICompletionsTelemetryReporters = createServiceIdentifier<ICompletionsTelemetryReporters>('ICompletionsTelemetryReporters'); +export interface ICompletionsTelemetryReporters { + readonly _serviceBrand: undefined; + getReporter(accessor: ServicesAccessor, store?: TelemetryStore): CopilotTelemetryReporter | undefined; + getEnhancedReporter(accessor: ServicesAccessor): CopilotTelemetryReporter | undefined; + getFTReporter(accessor: ServicesAccessor): CopilotTelemetryReporter | undefined; + setReporter(reporter: CopilotTelemetryReporter): void; + setEnhancedReporter(reporter: CopilotTelemetryReporter): void; + setFTReporter(reporter: CopilotTelemetryReporter): void; + deactivate(): Promise<void>; +} + +export class TelemetryReporters implements ICompletionsTelemetryReporters { + declare _serviceBrand: undefined; + + private reporter: CopilotTelemetryReporter | undefined; + private reporterEnhanced: CopilotTelemetryReporter | undefined; + private reporterFT: CopilotTelemetryReporter | undefined; + + getReporter(accessor: ServicesAccessor, store = TelemetryStore.Standard): CopilotTelemetryReporter | undefined { + return isEnhanced(store) ? this.getEnhancedReporter(accessor) : this.reporter; + } + getEnhancedReporter(accessor: ServicesAccessor): CopilotTelemetryReporter | undefined { + // Callers should do this check themselves as they may need to behave differently + // if we are not sending enhanced telemetry. The guard here is a backstop. + // Note: if the decision about what telemetry to send when the user is opted-out + // becomes more nuanced, we may need to drop this backstop. + if (shouldSendEnhanced(accessor)) { + return this.reporterEnhanced; + } + return undefined; + } + + getFTReporter(accessor: ServicesAccessor): CopilotTelemetryReporter | undefined { + return undefined; + } + + setReporter(reporter: CopilotTelemetryReporter): void { + this.reporter = reporter; + } + setEnhancedReporter(reporter: CopilotTelemetryReporter): void { + this.reporterEnhanced = reporter; + } + + setFTReporter(reporter: CopilotTelemetryReporter): void { + this.reporterFT = reporter; + } + + /** + * Synchronously unassign all reporters and asynchronously shut them down. + */ + async deactivate(): Promise<void> { + const reporters = [this.reporter, this.reporterEnhanced, this.reporterFT]; + this.reporter = this.reporterEnhanced = this.reporterFT = undefined; + await Promise.all(reporters.map(r => r?.dispose())); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/telemetry/userConfig.ts b/completions-sample-code/vscode-node/lib/src/telemetry/userConfig.ts new file mode 100644 index 0000000..28ed42c --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/telemetry/userConfig.ts @@ -0,0 +1,79 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../../../../../platform/authentication/common/authentication'; +import { CopilotToken } from '../../../../../../platform/authentication/common/copilotToken'; +import { createServiceIdentifier } from '../../../../../../util/common/services'; +import { Disposable } from '../../../../../../util/vs/base/common/lifecycle'; +import { onCopilotToken } from '../auth/copilotTokenNotifier'; + +interface UserConfigProperties { + copilot_trackingId: string; + organizations_list?: string; + enterprise_list?: string; + sku?: string; +} + +function propertiesFromCopilotToken(copilotToken: Omit<CopilotToken, 'token'>): UserConfigProperties | undefined { + const trackingId = copilotToken.getTokenValue('tid'); + const organizationsList = copilotToken.organizationList; + const enterpriseList = copilotToken.enterpriseList; + const sku = copilotToken.getTokenValue('sku'); + + if (!trackingId) { return; } + // The tracking id is also updated in reporters directly + // in the AppInsightsReporter class and set in the `ai.user.id` tag. + const props: UserConfigProperties = { copilot_trackingId: trackingId }; + if (organizationsList) { props.organizations_list = organizationsList.toString(); } + if (enterpriseList) { props.enterprise_list = enterpriseList.toString(); } + if (sku) { props.sku = sku; } + return props; +} + +export const ICompletionsTelemetryUserConfigService = createServiceIdentifier<ICompletionsTelemetryUserConfigService>('ICompletionsTelemetryUserConfigService'); +export interface ICompletionsTelemetryUserConfigService { + readonly _serviceBrand: undefined; + getProperties(): Partial<UserConfigProperties>; + trackingId: string | undefined; + optedIn: boolean; + ftFlag: string; +} + +export class TelemetryUserConfig extends Disposable implements ICompletionsTelemetryUserConfigService { + declare _serviceBrand: undefined; + #properties: Partial<UserConfigProperties> = {}; + optedIn = false; + ftFlag = ''; + + constructor( + @IAuthenticationService authenticationService: IAuthenticationService + ) { + super(); + + this._register(onCopilotToken(authenticationService, copilotToken => this.updateFromToken(copilotToken))); + + const maybeToken = authenticationService.copilotToken; + if (maybeToken) { + this.updateFromToken(maybeToken); + } + } + + getProperties() { + return this.#properties; + } + + get trackingId() { + return this.#properties.copilot_trackingId; + } + + updateFromToken(copilotToken: Omit<CopilotToken, 'token'>) { + const properties = propertiesFromCopilotToken(copilotToken); + if (properties) { + this.#properties = properties; + this.optedIn = copilotToken.getTokenValue('rt') === '1'; + this.ftFlag = copilotToken.getTokenValue('ft') ?? ''; + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/changeTracker.test.ts b/completions-sample-code/vscode-node/lib/src/test/changeTracker.test.ts new file mode 100644 index 0000000..b1b7f99 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/changeTracker.test.ts @@ -0,0 +1,49 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ChangeTracker } from '../changeTracker'; +import { createLibTestingContext } from './context'; +import { createTextDocument } from './textDocument'; + +suite('ChangeTracker test suite', function () { + const accessor = createLibTestingContext().createTestingAccessor(); + let clock: sinon.SinonFakeTimers; + setup(function () { + clock = sinon.useFakeTimers(); + }); + teardown(function () { + clock.restore(); + }); + test('It calls pushed actions after the timeout', async function () { + const document = createTextDocument('file:///foo.ts', 'typescript', 0, ''); + const tracker = accessor.get(IInstantiationService).createInstance(ChangeTracker, document.uri, 100); + let called = false; + tracker.push(() => { + called = true; + }, 10); + assert.strictEqual(called, false); + await clock.tickAsync(30); + assert.strictEqual(called, true); + }); + + test('It refuses new actions if already disposed', async function () { + const document = createTextDocument('file:///foo.ts', 'typescript', 0, ''); + const tracker = accessor.get(IInstantiationService).createInstance(ChangeTracker, document.uri, 100); + let called = 0; + tracker.push(() => { + called = 1; + }, 10); + await clock.tickAsync(30); + assert.throws(() => { + tracker.push(() => { + called = 2; + }, 100); + }); + assert.strictEqual(called, 1); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/completionNotifier.test.ts b/completions-sample-code/vscode-node/lib/src/test/completionNotifier.test.ts new file mode 100644 index 0000000..24f20cc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/completionNotifier.test.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import Sinon from 'sinon'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { CompletionNotifier, CompletionRequestedEvent } from '../completionNotifier'; +import { CompletionState, createCompletionState } from '../completionState'; +import { TelemetryWithExp } from '../telemetry'; +import { createLibTestingContext } from './context'; +import { createTextDocument } from './textDocument'; + +suite('Completion Notifier', function () { + let accessor: ServicesAccessor; + let notifier: CompletionNotifier; + let completionState: CompletionState; + let telemetryData: TelemetryWithExp; + + let clock: Sinon.SinonFakeTimers; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + const instantiationService = accessor.get(IInstantiationService); + notifier = instantiationService.createInstance(CompletionNotifier); + + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'const x = '); + const position = { line: 0, character: 10 }; + completionState = createCompletionState(textDocument, position); + telemetryData = TelemetryWithExp.createEmptyConfigForTesting(); + + clock = Sinon.useFakeTimers(); + }); + + teardown(function () { + clock.restore(); + }); + + test('should notify about requests', function () { + let notifiedEvent: CompletionRequestedEvent | undefined; + const disposable = notifier.onRequest((event: CompletionRequestedEvent) => { + notifiedEvent = event; + }); + + for (let i = 0; i < 3; i++) { + const completionId = `test-completion-id-${i}`; + notifier.notifyRequest(completionState, completionId, telemetryData); + assert.ok(notifiedEvent, 'Expected event to be notified'); + assert.strictEqual(notifiedEvent.completionId, completionId); + assert.strictEqual(notifiedEvent.completionState, completionState); + assert.strictEqual(notifiedEvent.telemetryData, telemetryData); + notifiedEvent = undefined; // Reset for each iteration + } + + disposable.dispose(); + }); + + test('should not propagate errors from listeners', function () { + // The telemetryCatch wrapper should handle errors, so the test should not throw + let errorThrown = false; + const disposable = notifier.onRequest(() => { + throw new Error('Test error from listener'); + }); + + try { + notifier.notifyRequest(completionState, 'test-completion-id', telemetryData); + // If we reach here, the error was caught and handled properly + } catch (error) { + errorThrown = true; + } + + assert.strictEqual(errorThrown, false, 'Error should be caught and not propagated'); + disposable.dispose(); + }); + + test('should dispose listeners', function () { + let requestCount = 0; + + const requestDisposable = notifier.onRequest(() => { + requestCount++; + }); + + // Dispose listeners before making any requests + requestDisposable.dispose(); + + // Make a request - should not trigger any listeners + notifier.notifyRequest(completionState, 'test-completion-id', telemetryData); + + assert.strictEqual(requestCount, 0, 'Request listener should be disposed'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/completionState.test.ts b/completions-sample-code/vscode-node/lib/src/test/completionState.test.ts new file mode 100644 index 0000000..a3d49d8 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/completionState.test.ts @@ -0,0 +1,244 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { TextEdit } from '../../../types/src/index'; +import { createCompletionState } from '../completionState'; +import { IntelliSenseInsertion } from '../textDocument'; +import { createTextDocument } from './textDocument'; + +suite('CompletionState', function () { + test('position unchanged when before edit range', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld'); + const position = { line: 0, character: 2 }; + const edit: TextEdit = { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }; + const completionState = createCompletionState(textDocument, position); + + const newState = completionState.applyEdits([edit]); + assert.deepStrictEqual(newState.position, position); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + assert.deepStrictEqual(newState.textDocument.getText(), 'hello\neveryone'); + assert.deepStrictEqual(newState.editsWithPosition.length, 1); + }); + + test('position adjusts when within edit range', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld'); + const position = { line: 1, character: 2 }; + const edit: TextEdit = { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }; + const completionState = createCompletionState(textDocument, position); + + const newState = completionState.applyEdits([edit]); + assert.deepStrictEqual(newState.position, { line: 1, character: 8 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'hello\neveryone'); + assert.deepStrictEqual(newState.editsWithPosition.length, 1); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('position at exact start of edit range gets moved to end of edit', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld'); + const position = { line: 1, character: 0 }; + const edit: TextEdit = { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }; + const completionState = createCompletionState(textDocument, position); + + const newState = completionState.applyEdits([edit]); + assert.deepStrictEqual(newState.position, { line: 1, character: 8 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'hello\neveryone'); + assert.deepStrictEqual(newState.editsWithPosition.length, 1); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('position after edit range adjusts by edit length difference', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld! How are you?'); + const position = { line: 1, character: 12 }; + const edit: TextEdit = { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }; + const completionState = createCompletionState(textDocument, position); + + const newState = completionState.applyEdits([edit]); + assert.deepStrictEqual(newState.position, { line: 1, character: 15 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'hello\neveryone! How are you?'); + assert.deepStrictEqual(newState.editsWithPosition.length, 1); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('can apply multiple edits', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld! How are you?'); + const position = { line: 1, character: 12 }; + const edits: TextEdit[] = [ + { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }, + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 5 }, + }, + newText: 'hi', + }, + ]; + const completionState = createCompletionState(textDocument, position); + + const newState = completionState.applyEdits(edits); + assert.deepStrictEqual(newState.position, { line: 1, character: 15 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'hi\neveryone! How are you?'); + assert.deepStrictEqual(newState.editsWithPosition.length, 2); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('can apply multiple edits in different calls', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld! How are you?'); + const position = { line: 1, character: 12 }; + const completionState = createCompletionState(textDocument, position); + + const intermediateState = completionState.applyEdits([ + { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }, + ]); + const newState = intermediateState.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 5 }, + }, + newText: 'hi', + }, + ]); + assert.deepStrictEqual(newState.position, { line: 1, character: 15 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'hi\neveryone! How are you?'); + assert.deepStrictEqual(newState.editsWithPosition.length, 2); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('selectedCompletionInfo is stored on its own, but applied as a normal edit', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'const person = Person.'); + const position = { line: 0, character: 22 }; + const completionState = createCompletionState(textDocument, position); + + const selectedCompletionInfo: IntelliSenseInsertion = { + text: 'getName', + range: { + start: { line: 0, character: 22 }, + end: { line: 0, character: 22 }, + }, + }; + + const newState = completionState.addSelectedCompletionInfo(selectedCompletionInfo); + assert.deepStrictEqual(newState.position, { line: 0, character: 29 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'const person = Person.getName'); + assert.deepStrictEqual(newState.editsWithPosition.length, 1); + assert.deepStrictEqual(newState.editsWithPosition[0].source, 'selectedCompletionInfo'); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('selectedCompletionInfo can only be applied once', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'const person = Person.'); + const position = { line: 0, character: 22 }; + const completionState = createCompletionState(textDocument, position); + + const selectedCompletionInfo: IntelliSenseInsertion = { + text: 'getName', + range: { + start: { line: 0, character: 22 }, + end: { line: 0, character: 22 }, + }, + }; + + const newState = completionState.addSelectedCompletionInfo(selectedCompletionInfo); + assert.throws(() => { + newState.addSelectedCompletionInfo(selectedCompletionInfo); + }); + }); + + test('selectedCompletionInfo combined with other edits', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'const person = Person.'); + const position = { line: 0, character: 22 }; + const completionState = createCompletionState(textDocument, position); + const selectedCompletionInfo: IntelliSenseInsertion = { + text: 'getName', + range: { + start: { line: 0, character: 22 }, + end: { line: 0, character: 22 }, + }, + }; + + const intermediateState = completionState.addSelectedCompletionInfo(selectedCompletionInfo); + + const speculativeEdit: TextEdit = { + newText: '()', + range: { + start: intermediateState.position, + end: intermediateState.position, + }, + }; + + const newState = intermediateState.applyEdits([speculativeEdit]); + assert.deepStrictEqual(newState.position, { line: 0, character: 31 }); + assert.deepStrictEqual(newState.textDocument.getText(), 'const person = Person.getName()'); + assert.deepStrictEqual(newState.editsWithPosition.length, 2); + assert.deepStrictEqual(newState.editsWithPosition[0].source, 'selectedCompletionInfo'); + assert.deepStrictEqual(newState.originalPosition, position); + assert.deepStrictEqual(newState.originalOffset, textDocument.offsetAt(position)); + }); + + test('updating position does not affect edits', function () { + const textDocument = createTextDocument('file:///test.ts', 'typescript', 1, 'hello\nworld'); + const position = { line: 0, character: 2 }; + const edit: TextEdit = { + range: { + start: { line: 1, character: 0 }, + end: { line: 1, character: 5 }, + }, + newText: 'everyone', + }; + const completionState = createCompletionState(textDocument, position); + const newState = completionState.applyEdits([edit]); + const updatedState = newState.updatePosition({ line: 0, character: 5 }); + + assert.deepStrictEqual(updatedState.position, { line: 0, character: 5 }); + assert.deepStrictEqual(updatedState.textDocument.getText(), 'hello\neveryone'); + assert.deepStrictEqual(updatedState.editsWithPosition.length, 1); + assert.deepStrictEqual(updatedState.originalPosition, position); + assert.deepStrictEqual(updatedState.originalOffset, textDocument.offsetAt(position)); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/completionsPrompt.ts b/completions-sample-code/vscode-node/lib/src/test/completionsPrompt.ts new file mode 100644 index 0000000..74ee924 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/completionsPrompt.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationTokenSource, Position } from 'vscode-languageserver-protocol'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + CompletionRequestData, + CompletionRequestDocument, +} from '../prompt/completionsPromptFactory/componentsCompletionsPromptFactory'; +import { CodeSnippetWithId, TraitWithId } from '../prompt/contextProviders/contextItemSchemas'; +import { TelemetryWithExp } from '../telemetry'; + +export function createCompletionRequestData( + accessor: ServicesAccessor, + doc: CompletionRequestDocument, + position: Position, + codeSnippets?: CodeSnippetWithId[], + traits?: TraitWithId[], + turnOffSimilarFiles?: boolean, + suffixMatchThreshold?: number, + maxPromptLength?: number +): CompletionRequestData { + return { + document: doc, + position, + telemetryData: TelemetryWithExp.createEmptyConfigForTesting(), + cancellationToken: new CancellationTokenSource().token, + codeSnippets, + traits, + turnOffSimilarFiles, + suffixMatchThreshold, + maxPromptTokens: maxPromptLength ?? 1000, + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/test/config.test.ts b/completions-sample-code/vscode-node/lib/src/test/config.test.ts new file mode 100644 index 0000000..9bf5dee --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/config.test.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { + ConfigKey, + DefaultsOnlyConfigProvider, + InMemoryConfigProvider, + getConfigDefaultForKey, + getConfigKeyRecursively, +} from '../config'; + +suite('getConfig', function () { + for (const key of Object.values(ConfigKey)) { + test(`has default for ${key}`, function () { + // No news is good news + getConfigDefaultForKey(key); + }); + } +}); + +suite('getConfigKeyRecursively', function () { + test('handles arbitrary dots', function () { + const config = { + 'a.b.c': { 'd.e': 'value' }, + }; + assert.strictEqual(getConfigKeyRecursively(config, 'a.b.c.d.e'), 'value'); + }); +}); + +suite('InMemoryConfigProvider', function () { + test('allows setting and getting config values', function () { + const configProvider = new InMemoryConfigProvider(new DefaultsOnlyConfigProvider()); + configProvider.setConfig(ConfigKey.DebugOverrideEngine, 'test'); + assert.strictEqual(configProvider.getConfig(ConfigKey.DebugOverrideEngine), 'test'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/context.ts b/completions-sample-code/vscode-node/lib/src/test/context.ts new file mode 100644 index 0000000..5f0840d --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/context.ts @@ -0,0 +1,153 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DocumentSelector } from 'vscode-languageserver-protocol/lib/common/protocol'; +import { ILanguageContextProviderService } from '../../../../../../platform/languageContextProvider/common/languageContextProviderService'; +import { NullLanguageContextProviderService } from '../../../../../../platform/languageContextProvider/common/nullLanguageContextProviderService'; +import { TestingServiceCollection } from '../../../../../../platform/test/node/services'; +import { SyncDescriptor } from '../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { createExtensionTestingServices } from '../../../../../test/vscode-node/services'; +import { CompletionsTelemetryServiceBridge, ICompletionsTelemetryService } from '../../../bridge/src/completionsTelemetryServiceBridge'; +import { DocumentContext } from '../../../types/src'; +import { ICompletionsCopilotTokenManager } from '../auth/copilotTokenManager'; +import { ICompletionsCitationManager, NoOpCitationManager } from '../citationManager'; +import { CompletionNotifier, ICompletionsNotifierService } from '../completionNotifier'; +import { + DefaultsOnlyConfigProvider, ICompletionsConfigProvider, + ICompletionsEditorAndPluginInfo, + InMemoryConfigProvider +} from '../config'; +import { ICompletionsUserErrorNotifierService, UserErrorNotifier } from '../error/userErrorNotifier'; +import { Features } from '../experiments/features'; +import { ICompletionsFeaturesService } from '../experiments/featuresService'; +import { FileReader, ICompletionsFileReaderService } from '../fileReader'; +import { ICompletionsFileSystemService } from '../fileSystem'; +import { AsyncCompletionManager, ICompletionsAsyncManagerService } from '../ghostText/asyncCompletions'; +import { CompletionsCache, ICompletionsCacheService } from '../ghostText/completionsCache'; +import { ConfigBlockModeConfig, ICompletionsBlockModeConfig } from '../ghostText/configBlockMode'; +import { CurrentGhostText, ICompletionsCurrentGhostText } from '../ghostText/current'; +import { ICompletionsLastGhostText, LastGhostText } from '../ghostText/last'; +import { ICompletionsSpeculativeRequestCache, SpeculativeRequestCache } from '../ghostText/speculativeRequestCache'; +import { LocalFileSystem } from '../localFileSystem'; +import { ICompletionsLogTargetService } from '../logger'; +import { ICompletionsFetcherService } from '../networking'; +import { ICompletionsNotificationSender } from '../notificationSender'; +import { AvailableModelsManager, ICompletionsModelManagerService } from '../openai/model'; +import { ICompletionsStatusReporter, NoOpStatusReporter } from '../progress'; +import { + CompletionsPromptFactory, ICompletionsPromptFactoryService +} from '../prompt/completionsPromptFactory/completionsPromptFactory'; +import { ContextProviderBridge, ICompletionsContextProviderBridgeService } from '../prompt/components/contextProviderBridge'; +import { + CachedContextProviderRegistry, + DefaultContextProvidersContainer, ICompletionsContextProviderRegistryService, + ICompletionsDefaultContextProviders, + MutableContextProviderRegistry +} from '../prompt/contextProviderRegistry'; +import { ContextProviderStatistics, ICompletionsContextProviderService } from '../prompt/contextProviderStatistics'; +import { EmptyRecentEditsProvider } from '../prompt/recentEdits/emptyRecentEditsProvider'; +import { ICompletionsRecentEditsProviderService } from '../prompt/recentEdits/recentEditsProvider'; +import { ICompletionsTelemetryReporters, TelemetryReporters } from '../telemetry'; +import { ICompletionsTelemetryUserConfigService, TelemetryUserConfig } from '../telemetry/userConfig'; +import { ICompletionsTextDocumentManagerService } from '../textDocumentManager'; +import { ICompletionsPromiseQueueService } from '../util/promiseQueue'; +import { ICompletionsRuntimeModeService, RuntimeMode } from '../util/runtimeMode'; +import { FakeCopilotTokenManager } from './copilotTokenManager'; +import { NoFetchFetcher } from './fetcher'; +import { TestPromiseQueue } from './telemetry'; +import { TestNotificationSender } from './testHelpers'; +import { TestTextDocumentManager } from './textDocument'; + +class NullLog implements ICompletionsLogTargetService { + declare _serviceBrand: undefined; + logIt(..._: unknown[]) { } +} + +/** + * Baseline for a context. Tests should prefer the specific variants outlined below. + * + * @see createLibTestingContext + * @see createExtensionTestingContext + * @see createAgentTestingContext + */ +export function _createBaselineContext(serviceCollection: TestingServiceCollection, configProvider: InMemoryConfigProvider): TestingServiceCollection { + serviceCollection.set(ILanguageContextProviderService, new NullLanguageContextProviderService()); + + serviceCollection.define(ICompletionsLogTargetService, new NullLog()); + serviceCollection.define(ICompletionsCacheService, new CompletionsCache()); + serviceCollection.define(ICompletionsConfigProvider, configProvider); + serviceCollection.define(ICompletionsRuntimeModeService, new RuntimeMode({ debug: false, verboseLogging: false, testMode: true, simulation: false })); + serviceCollection.define(ICompletionsSpeculativeRequestCache, new SpeculativeRequestCache()); + serviceCollection.define(ICompletionsLastGhostText, new LastGhostText()); + serviceCollection.define(ICompletionsCurrentGhostText, new CurrentGhostText()); + serviceCollection.define(ICompletionsStatusReporter, new NoOpStatusReporter()); + serviceCollection.define(ICompletionsCitationManager, new NoOpCitationManager()); + serviceCollection.define(ICompletionsNotificationSender, new TestNotificationSender()); + serviceCollection.define(ICompletionsTelemetryReporters, new TelemetryReporters()); + serviceCollection.define(ICompletionsCopilotTokenManager, new FakeCopilotTokenManager()); + serviceCollection.define(ICompletionsFeaturesService, new SyncDescriptor(Features)); + serviceCollection.define(ICompletionsTelemetryService, new SyncDescriptor(CompletionsTelemetryServiceBridge)); + serviceCollection.define(ICompletionsNotifierService, new SyncDescriptor(CompletionNotifier)); + serviceCollection.define(ICompletionsBlockModeConfig, new SyncDescriptor(ConfigBlockModeConfig)); + serviceCollection.define(ICompletionsRecentEditsProviderService, new EmptyRecentEditsProvider()); + serviceCollection.define(ICompletionsUserErrorNotifierService, new SyncDescriptor(UserErrorNotifier)); + + serviceCollection.define(ICompletionsFileReaderService, new SyncDescriptor(FileReader)); + serviceCollection.define(ICompletionsTelemetryUserConfigService, new SyncDescriptor(TelemetryUserConfig)); + serviceCollection.define(ICompletionsModelManagerService, new SyncDescriptor(AvailableModelsManager, [false])); + serviceCollection.define(ICompletionsAsyncManagerService, new SyncDescriptor(AsyncCompletionManager)); + serviceCollection.define(ICompletionsContextProviderBridgeService, new SyncDescriptor(ContextProviderBridge)); + serviceCollection.define(ICompletionsPromiseQueueService, new TestPromiseQueue()); + + //ctx.set(FileSearch, new TestingFileSearch()); + serviceCollection.define(ICompletionsPromptFactoryService, new SyncDescriptor(CompletionsPromptFactory)); + serviceCollection.define(ICompletionsContextProviderService, new ContextProviderStatistics()); + serviceCollection.define(ICompletionsContextProviderRegistryService, + new SyncDescriptor(CachedContextProviderRegistry, [MutableContextProviderRegistry, (_: unknown, documentSelector: DocumentSelector, documentContext: DocumentContext) => { + if (documentSelector.find(ds => ds === '*')) { + return 1; + } + return documentSelector.find(ds => typeof ds !== 'string' && ds.language === documentContext.languageId) + ? 10 + : 0; + }]) + ); + + return serviceCollection; +} + +/** + * @returns a context suitable for `lib` tests. + */ +export function createLibTestingContext() { + let serviceCollection = createExtensionTestingServices(); + serviceCollection = _createBaselineContext(serviceCollection, new InMemoryConfigProvider(new DefaultsOnlyConfigProvider())); + + serviceCollection.define(ICompletionsFetcherService, new NoFetchFetcher()); + serviceCollection.define(ICompletionsEditorAndPluginInfo, new LibTestsEditorInfo()); + serviceCollection.define(ICompletionsTextDocumentManagerService, new SyncDescriptor(TestTextDocumentManager)); + serviceCollection.define(ICompletionsFileSystemService, new LocalFileSystem()); + serviceCollection.define(ICompletionsDefaultContextProviders, new DefaultContextProvidersContainer()); + + return serviceCollection; +} + +export class LibTestsEditorInfo implements ICompletionsEditorAndPluginInfo { + declare _serviceBrand: undefined; + constructor( + readonly editorPluginInfo = { name: 'lib-tests-plugin', version: '2' }, + readonly editorInfo = { name: 'lib-tests-editor', version: '1' }, + readonly relatedPluginInfo = [{ name: 'lib-tests-related-plugin', version: '3' }] + ) { } + getEditorInfo() { + return this.editorInfo; + } + getEditorPluginInfo() { + return this.editorPluginInfo; + } + getRelatedPluginInfo() { + return this.relatedPluginInfo; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/copilotTokenManager.ts b/completions-sample-code/vscode-node/lib/src/test/copilotTokenManager.ts new file mode 100644 index 0000000..d38107e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/copilotTokenManager.ts @@ -0,0 +1,42 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotToken, createTestExtendedTokenInfo, type ExtendedTokenInfo } from '../../../../../../platform/authentication/common/copilotToken'; +import { ICompletionsCopilotTokenManager } from '../auth/copilotTokenManager'; + +// Buffer to allow refresh to happen successfully +export class FakeCopilotTokenManager implements ICompletionsCopilotTokenManager { + declare _serviceBrand: undefined; + private _token: CopilotToken; + + constructor() { + this._token = FakeCopilotTokenManager.createTestCopilotToken({ token: 'tid=test;rt=1' }); + } + + get token(): CopilotToken | undefined { + return this._token; + } + + primeToken(): Promise<boolean> { + return Promise.resolve(true); + } + + async getToken(): Promise<CopilotToken> { + return this._token; + } + + resetToken(httpError?: number): void { + } + + getLastToken(): Omit<CopilotToken, 'token'> | undefined { + return this._token; + } + + private static readonly REFRESH_BUFFER_SECONDS = 60; + private static createTestCopilotToken(overrides?: Partial<Omit<ExtendedTokenInfo, 'expires_at'>>): CopilotToken { + const expires_at = Date.now() + ((overrides?.refresh_in ?? 0) + FakeCopilotTokenManager.REFRESH_BUFFER_SECONDS) * 1000; + return new CopilotToken(createTestExtendedTokenInfo({ expires_at, ...overrides })); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/fetcher.ts b/completions-sample-code/vscode-node/lib/src/test/fetcher.ts new file mode 100644 index 0000000..996e4fc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/fetcher.ts @@ -0,0 +1,168 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotNamedAnnotationList } from '../../../../../../platform/completions-core/common/openai/copilotAnnotations'; +import { FetchOptions, IAbortController, ICompletionsFetcherService, IHeaders, Response } from '../networking'; + +type HeadersParameter = { [key: string]: string }; + +export function createFakeResponse(statusCode: number, response?: string, headers?: HeadersParameter) { + const fakeHeaders = new FakeHeaders(); + fakeHeaders.set('x-github-request-id', '1'); + for (const [key, value] of Object.entries(headers || {})) { + fakeHeaders.set(key, value); + } + return Response.fromText( + statusCode, + 'status text', + fakeHeaders, + response ?? '', + 'test-stub' + ); +} + +export function createFakeJsonResponse(statusCode: number, response: string | object, headers?: HeadersParameter) { + let text: string; + if (typeof response === 'string') { + text = response; + } else { + text = JSON.stringify(response); + } + return createFakeResponse(statusCode, text, Object.assign({ 'content-type': 'application/json' }, headers)); +} + +export function createFakeStreamResponse(body: string): Response { + return Response.fromText( + 200, + 'Success', + new FakeHeaders(), + body, + 'test-stub' + ); +} + +export function createFakeCompletionResponse( + completionText: string | string[], + options?: { annotations?: CopilotNamedAnnotationList } +): Response { + const now = Math.floor(Date.now() / 1000); + if (typeof completionText === 'string') { + completionText = [completionText]; + } + const choices = completionText.map((text, i) => ({ + text, + index: i, + finishReason: 'stop', + logprobs: null, + copilot_annotations: options?.annotations, + p: 'aaaaaa', + })); + const responseObject = { + id: 'cmpl-AaZz1234', + created: now, + model: 'unit-test', + choices, + }; + const responseLines = [JSON.stringify(responseObject), `[DONE]`]; + return createFakeStreamResponse(responseLines.map(l => `data: ${l}\n`).join('')); +} + +export function fakeCodeReference( + startOffset: number = 0, + stopOffset: number = 1, + license: string = 'MIT', + url: string = 'https://github.com/github/example' +): CopilotNamedAnnotationList { + return { + ip_code_citations: [ + { + id: 5, + start_offset: startOffset, + stop_offset: stopOffset, + details: { + citations: [ + { + url, + license, + }, + ], + }, + }, + ], + }; +} + +export abstract class FakeFetcher implements ICompletionsFetcherService { + declare _serviceBrand: undefined; + + abstract fetch(url: string, options: FetchOptions): Promise<Response>; + getImplementation(): ICompletionsFetcherService | Promise<ICompletionsFetcherService> { + return this; + } + disconnectAll(): Promise<unknown> { + throw new Error('Method not implemented.'); + } +} + +type FakeResponseGenerator = (url: string, options: FetchOptions) => Response | Promise<Response>; +const SuccessResponseGenerator: FakeResponseGenerator = () => createFakeResponse(200); + +export class StaticFetcher extends FakeFetcher { + constructor(private createResponse: FakeResponseGenerator = SuccessResponseGenerator) { + super(); + } + + headerBuffer: { [name: string]: string } | undefined; + + fetch(url: string, options: FetchOptions): Promise<Response> { + this.headerBuffer = options.headers; + return Promise.resolve(this.createResponse(url, options)); + } +} + +export class NoFetchFetcher extends FakeFetcher { + fetch(url: string, options: FetchOptions): Promise<Response> { + throw new Error('NoFetchFetcher does not support fetching'); + } +} + +class FakeHeaders implements IHeaders { + private readonly headers: Map<string, string> = new Map(); + + append(name: string, value: string): void { + this.headers.set(name.toLowerCase(), value); + } + delete(name: string): void { + this.headers.delete(name.toLowerCase()); + } + get(name: string): string | null { + return this.headers.get(name.toLowerCase()) ?? null; + } + has(name: string): boolean { + return this.headers.has(name.toLowerCase()); + } + set(name: string, value: string): void { + this.headers.set(name.toLowerCase(), value); + } + entries(): Iterator<[string, string]> { + return this.headers.entries(); + } + keys(): Iterator<string> { + return this.headers.keys(); + } + values(): Iterator<string> { + return this.headers.values(); + } + [Symbol.iterator](): Iterator<[string, string]> { + return this.headers.entries(); + } +} + +export class FakeAbortController implements IAbortController { + readonly signal = { aborted: false, addEventListener: () => { }, removeEventListener: () => { } }; + abort(): void { + this.signal.aborted = true; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/fileReader.test.ts b/completions-sample-code/vscode-node/lib/src/test/fileReader.test.ts new file mode 100644 index 0000000..d744615 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/fileReader.test.ts @@ -0,0 +1,85 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { FileReader } from '../fileReader'; +import { ICompletionsFileSystemService } from '../fileSystem'; +import { ICompletionsTextDocumentManagerService } from '../textDocumentManager'; +import { createLibTestingContext } from './context'; +import { FakeFileSystem } from './filesystem'; +import { TestTextDocumentManager } from './textDocument'; + +suite('File Reader', function () { + let sandbox: sinon.SinonSandbox; + let accessor: ServicesAccessor; + + setup(function () { + sandbox = sinon.createSandbox(); + const serviceCollection = createLibTestingContext(); + serviceCollection.define( + ICompletionsFileSystemService, + new FakeFileSystem({ + '/test.ts': FakeFileSystem.file('const foo', { ctime: 0, mtime: 0, size: 0.1 * 1024 * 1024 }), // .1MB + '/empty.ts': '', + '/large.ts': FakeFileSystem.file('very large file', { ctime: 0, mtime: 0, size: 1.1 * 1024 * 1024 }), // 1.1MB + }) + ); + accessor = serviceCollection.createTestingAccessor(); + }); + + teardown(function () { + sandbox.restore(); + }); + + test('reads file from text document manager', async function () { + const tdm = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + tdm.setTextDocument('file:///test.js', 'javascript', 'const abc ='); + const reader = accessor.get(IInstantiationService).createInstance(FileReader); + + const docResult = await reader.getOrReadTextDocument({ uri: 'file:///test.js' }); + + assert.deepStrictEqual(docResult.status, 'valid'); + assert.deepStrictEqual(docResult.document?.getText(), 'const abc ='); + assert.deepStrictEqual(docResult.document?.detectedLanguageId, 'javascript'); + }); + + test('reads file from file system', async function () { + const reader = accessor.get(IInstantiationService).createInstance(FileReader); + + const docResult = await reader.getOrReadTextDocument({ uri: 'file:///test.ts' }); + + assert.deepStrictEqual(docResult.status, 'valid'); + assert.deepStrictEqual(docResult.document?.getText(), 'const foo'); + assert.deepStrictEqual(docResult.document?.detectedLanguageId, 'typescript'); + }); + + test('reads notfound from non existing file', async function () { + const reader = accessor.get(IInstantiationService).createInstance(FileReader); + + const docResult = await reader.getOrReadTextDocument({ uri: 'file:///UNKNOWN.ts' }); + + assert.deepStrictEqual(docResult.status, 'notfound'); + assert.deepStrictEqual(docResult.message, 'File not found'); + }); + + test('reads notfound for file too large', async function () { + const reader = accessor.get(IInstantiationService).createInstance(FileReader); + + const docResult = await reader.getOrReadTextDocument({ uri: 'file:///large.ts' }); + + assert.deepStrictEqual(docResult.status, 'notfound'); + assert.deepStrictEqual(docResult.message, 'File too large'); + }); + + test('reads empty files', async function () { + const reader = accessor.get(IInstantiationService).createInstance(FileReader); + const docResult = await reader.getOrReadTextDocument({ uri: 'file:///empty.ts' }); + + assert.deepStrictEqual(docResult.status, 'valid'); + assert.deepStrictEqual(docResult.document.getText(), ''); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/filesystem.ts b/completions-sample-code/vscode-node/lib/src/test/filesystem.ts new file mode 100644 index 0000000..fb27aad --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/filesystem.ts @@ -0,0 +1,202 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { dirname, join, normalize } from 'path'; +import { FileIdentifier, FileStat, FileType, ICompletionsFileSystemService } from '../fileSystem'; +import { getFsPath } from '../util/uri'; + +interface Exception extends Error { + errno?: unknown; + code?: unknown; + path?: unknown; + syscall?: unknown; + cause?: unknown; + toString(): string; +} + +abstract class FakeFileNode { + abstract readonly stats: FileStat; + abstract readonly isDir: boolean; +} + +type FakeFileEntries = { [key: string]: FakeFileNode }; + +class FakeFile extends FakeFileNode { + readonly isDir = false; + constructor( + readonly content: string, + readonly stats: FileStat + ) { + super(); + } +} + +class FakeDir extends FakeFileNode { + readonly isDir = true; + readonly entries: FakeFileEntries = {}; + constructor(readonly stats: FileStat) { + super(); + } +} + +export type FakeFileSystemConfig = { [key: string]: string | FakeFileNode | FakeFileSystemConfig }; + +/** + * A fake for FileSystem that returns content and stats for a set of files + * and folders configured for testing purposes. + * + * Accepts a configuration like the following: + * + * ```js + * { + * "/path/to/file": "file content", + * "/path/to/folder": { + * "file1": "file1 content", + * "file2": "file2 content", + * } + * } + * ``` + * + * It is also possible to control the results of `stat` by using `.file` and + * `.directory` to create fakes: + * + * ```js + * { + * "/bigFile.txt": FakeFileSystem.file({ctime: 0, mtime: 0, size: 1000000}), + * "/futureFolder": FakeFileSystem.directory({ + * ctime: Date.now() + 3600000, + * mtime: Date.now() + 3600000, + * size: 64}), + * } + * ``` + */ +export class FakeFileSystem implements ICompletionsFileSystemService { + declare _serviceBrand: undefined; + + private root: FakeDir; + + constructor(fileConfig: FakeFileSystemConfig) { + this.root = new FakeDir({ ctime: 0, mtime: 0, size: 0, type: FileType.Directory }); + this.createFiles('', fileConfig); + } + + private createFiles(parent: string, config: FakeFileSystemConfig): void { + for (const [key, value] of Object.entries(config)) { + const path = join(parent, key); + if (value instanceof FakeFileNode) { + this.mkdir(dirname(path)); + this.writeNode(path, value); + } else if (typeof value === 'string') { + this.mkdir(dirname(path)); + this.writeFile(path, value); + } else { + this.mkdir(path); + this.createFiles(path, value); + } + } + } + + /** Recursively creates directories in path */ + mkdir(path: string): void { + if (!this.getNode(this.root, this.pathParts(path), true, 'mkdir').isDir) { + throw this.noEntryError(`mkdir '${path}'`); + } + } + + writeFile(path: string, data: string): void { + this.writeNode(path, new FakeFile(data, { ctime: 0, mtime: 0, size: data.length, type: FileType.File })); + } + + private writeNode(path: string, node: FakeFileNode): void { + const parts = this.pathParts(path); + const filename = parts.pop() || ''; + const parent = this.getNode(this.root, parts, false, 'writeFile'); + if (!(parent instanceof FakeDir)) { + throw this.noEntryError(`writeFile '${path}'`); + } else if (parent.entries[filename]?.isDir) { + throw this.isDirectoryError(`open '${path}'`); + } + parent.entries[filename] = node; + } + + async readFileString(uri: FileIdentifier): Promise<string> { + const fsPath = getFsPath(uri) ?? '<invalid file URI>'; + const file = this.getNode(this.root, this.pathParts(fsPath), false, 'open'); + if (file.isDir) { + throw this.isDirectoryError(`open '${fsPath}'`); + } + return Promise.resolve((file as FakeFile).content); + } + + stat(uri: FileIdentifier): Promise<FileStat> { + return Promise.resolve(this.getNode(this.root, this.pathParts(getFsPath(uri)!), false, 'stat').stats); + } + + async readDirectory(uri: FileIdentifier): Promise<[string, FileType][]> { + const fsPath = getFsPath(uri) ?? '<invalid file URI>'; + const node = this.getNode(this.root, this.pathParts(fsPath), false, 'readDirectory'); + if (!(node instanceof FakeDir)) { + throw this.noEntryError(`readDirectory '${fsPath}'`); + } + return Promise.resolve( + Object.entries(node.entries).map(([name, entry]) => [ + name, + entry.isDir ? FileType.Directory : FileType.File, + ]) + ); + } + + private getNode(parent: FakeDir, parts: string[], createPath: boolean, command: string): FakeFileNode { + let current: FakeFileNode = parent; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (!(current instanceof FakeDir) || current.entries[part] === undefined) { + if (createPath && current instanceof FakeDir) { + current.entries[part] = new FakeDir({ ctime: 0, mtime: 0, size: 0, type: FileType.Directory }); + } else { + throw this.noEntryError(`${command} '${parts.join('/')}'`); + } + } + current = current.entries[part]; + } + return current; + } + + private pathParts(path: string): string[] { + const parts = normalize(path).split(/[\\/]+/); + if (parts[0] === '') { + parts.shift(); + } + if (parts[parts.length - 1] === '') { + parts.pop(); + } + return parts; + } + + private noEntryError(description: string): Error { + const err: Exception = new Error(`ENOENT: no such file or directory, ${description}`); + err.errno = -2; + err.code = 'ENOENT'; + return err; + } + + private isDirectoryError(description: string): Error { + const err: Exception = new Error(`EISDIR: illegal operation on a directory, ${description}`); + err.errno = -21; + err.code = 'EISDIR'; + return err; + } + + static file(content = '', stats?: Partial<FileStat>) { + return new FakeFile( + content, + Object.assign({ ctime: 0, mtime: 0, size: content.length, type: FileType.File }, stats) + ); + } + + static directory(stats?: Partial<FileStat>) { + return new FakeDir(Object.assign({ ctime: 0, mtime: 0, size: 0, type: FileType.Directory }, stats)); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/inlineCompletion.test.ts b/completions-sample-code/vscode-node/lib/src/test/inlineCompletion.test.ts new file mode 100644 index 0000000..7f71699 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/inlineCompletion.test.ts @@ -0,0 +1,189 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import Sinon from 'sinon'; +import { SyncDescriptor } from '../../../../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ResultType } from '../ghostText/ghostText'; +import { telemetryShown } from '../ghostText/telemetry'; +import { GhostText } from '../inlineCompletion'; +import { FetchOptions, ICompletionsFetcherService, Response } from '../networking'; +import { CompletionRequest, ICompletionsOpenAIFetcherService, LiveOpenAIFetcher } from '../openai/fetch'; +import { LocationFactory } from '../textDocument'; +import { Deferred, delay } from '../util/async'; +import { createLibTestingContext } from './context'; +import { createFakeCompletionResponse, StaticFetcher } from './fetcher'; +import { withInMemoryTelemetry } from './telemetry'; +import { createTextDocument } from './textDocument'; + +suite('getInlineCompletions()', function () { + function setupCompletion( + fetcher: ICompletionsFetcherService, + docText = 'function example() {\n\n}', + position = LocationFactory.position(1, 0), + languageId = 'typescript' + ) { + const serviceCollection = createLibTestingContext(); + const doc = createTextDocument('file:///example.ts', languageId, 1, docText); + serviceCollection.define(ICompletionsFetcherService, fetcher); + serviceCollection.define(ICompletionsOpenAIFetcherService, new SyncDescriptor(LiveOpenAIFetcher)); // gets results from static fetcher + const accessor = serviceCollection.createTestingAccessor(); + + // Setup closures with the state as default + function requestInlineCompletions(textDoc = doc, pos = position) { + const instaService = accessor.get(IInstantiationService); + const ghostText = instaService.createInstance(GhostText); + return ghostText.getInlineCompletions(textDoc, pos); + } + + return { + accessor, + doc, + position, + requestInlineCompletions, + }; + } + + test('Sends a speculative request when shown', async function () { + const firstCompletionText = '\tconst firstVar = 1;'; + const secondCompletionText = '\tconst secondVar = 2;'; + + const completionsDeferred = new Deferred<CompletionRequest>(); + const networkResponse = Sinon.stub<[string, FetchOptions], Response>().returns( + createFakeCompletionResponse('// not expected!') + ); + networkResponse.onFirstCall().returns(createFakeCompletionResponse(firstCompletionText)); + networkResponse.onSecondCall().callsFake((_url, opts) => { + completionsDeferred.resolve(opts.json as CompletionRequest); + return createFakeCompletionResponse(secondCompletionText); + }); + const { accessor, doc, position, requestInlineCompletions } = setupCompletion(new StaticFetcher(networkResponse)); + + const { reporter, result } = await withInMemoryTelemetry(accessor, async () => { + const firstResponse = await requestInlineCompletions(); + + assert.strictEqual(firstResponse?.length, 1); + assert.strictEqual(firstResponse[0].insertText, firstCompletionText); + telemetryShown(accessor, firstResponse[0]); + + // We're expecting 2 completion requests: one we explicitly requested, and a follow-up speculative request in the background. + return await completionsDeferred.promise; + }); + + const expectedPrefix = doc.getText({ start: { line: 0, character: 0 }, end: position }) + firstCompletionText; + assert.ok(result.prompt.endsWith(expectedPrefix), 'Expect first completion in second request'); + + const issuedTelemetry = reporter.eventsMatching(event => event.name === 'ghostText.issued'); + assert.strictEqual(issuedTelemetry.length, 2, `Expected 2 issued events, got ${issuedTelemetry.length}`); + + const speculativeTelemetry = reporter.eventsMatching( + event => event.name === 'ghostText.issued' && event.properties['reason'] === 'speculative' + ); + assert.ok(speculativeTelemetry.length === 1, 'Expected one speculative request'); + }); + + test('speculative requests apply completions the same as the editor and CLS', async function () { + const firstCompletion = ' const firstVar = 1;'; + const secondCompletion = '\n const secondVar = 2;'; + const completionsDeferred = new Deferred<void>(); + const networkResponse = Sinon.stub<[], Response>().returns(createFakeCompletionResponse('// not expected!')); + networkResponse.onFirstCall().returns(createFakeCompletionResponse(firstCompletion)); + networkResponse.onSecondCall().callsFake(() => { + completionsDeferred.resolve(); + return createFakeCompletionResponse(secondCompletion); + }); + const { accessor, doc, position, requestInlineCompletions } = setupCompletion( + new StaticFetcher(networkResponse), + 'function example() {\n \n}\n', + LocationFactory.position(1, 4) + ); + + const response = await requestInlineCompletions(); + + assert.strictEqual(response?.length, 1); + assert.strictEqual(response[0].insertText, firstCompletion); + assert.deepStrictEqual(response[0].range, LocationFactory.range(LocationFactory.position(1, 0), position)); + + telemetryShown(accessor, response[0]); + await completionsDeferred.promise; // Wait for speculative request to be sent + + const docv2 = createTextDocument( + doc.uri, + doc.clientLanguageId, + doc.version + 1, + `function example() {\n${firstCompletion}\n}\n` + ); + const position2 = LocationFactory.position(1, firstCompletion.length); + const response2 = await requestInlineCompletions(docv2, position2); + + assert.strictEqual(response2?.length, 1); + assert.strictEqual(response2[0].insertText, firstCompletion + secondCompletion); + assert.deepStrictEqual( + response2[0].range, + LocationFactory.range(LocationFactory.position(1, 0), LocationFactory.position(1, firstCompletion.length)) + ); + assert.strictEqual(response2[0].resultType, ResultType.Cache); + assert.strictEqual(networkResponse.callCount, 2); + }); + + test('does not send a speculative request if empty', async function () { + const { accessor, requestInlineCompletions } = setupCompletion( + new StaticFetcher(() => createFakeCompletionResponse('')) + ); + + const { reporter, result } = await withInMemoryTelemetry(accessor, () => { + return requestInlineCompletions(); + }); + + assert.strictEqual(result, undefined); + const issuedTelemetry = reporter.eventsMatching(event => event.name === 'ghostText.issued'); + assert.strictEqual(issuedTelemetry.length, 1, `Expected 1 issued events, got ${issuedTelemetry.length}`); + const speculativeTelemetry = reporter.eventsMatching( + event => event.name === 'ghostText.issued' && event.properties['reason'] === 'speculative' + ); + assert.ok(speculativeTelemetry.length === 0, 'Expected no speculative request'); + }); + + test('telemetryShown triggers speculative request only when shown', async function () { + const firstCompletionText = '\tconst firstVar = 1;'; + const secondCompletionText = '\tconst secondVar = 2;'; + const completionsDeferred = new Deferred<CompletionRequest>(); + const networkResponse = Sinon.stub<[string, FetchOptions], Response>().returns( + createFakeCompletionResponse('// not expected!') + ); + networkResponse.onFirstCall().returns(createFakeCompletionResponse(firstCompletionText)); + networkResponse.onSecondCall().callsFake((_url, opts) => { + completionsDeferred.resolve(opts.json as CompletionRequest); + return createFakeCompletionResponse(secondCompletionText); + }); + + const { accessor, requestInlineCompletions } = setupCompletion(new StaticFetcher(networkResponse)); + + const { reporter } = await withInMemoryTelemetry(accessor, async () => { + const firstResponse = await requestInlineCompletions(); + assert.strictEqual(firstResponse?.length, 1); + assert.strictEqual(firstResponse[0].insertText, firstCompletionText); + + // Verify speculative request is not made before shown + await delay(50); + assert.strictEqual(networkResponse.callCount, 1, 'Expected only the initial network call'); + + // Call telemetryShown to trigger speculative request + telemetryShown(accessor, firstResponse[0]); + + // Wait for speculative request to complete + return await completionsDeferred.promise; + }); + + assert.strictEqual(networkResponse.callCount, 2, 'Expected 2 network calls (original + speculative)'); + const shownTelemetry = reporter.eventsMatching(event => event.name === 'ghostText.shown'); + assert.strictEqual(shownTelemetry.length, 1, 'Expected one shown telemetry event'); + const speculativeTelemetry = reporter.eventsMatching( + event => event.name === 'ghostText.issued' && event.properties['reason'] === 'speculative' + ); + assert.ok(speculativeTelemetry.length === 1, 'Expected one speculative request'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/localFileSystem.test.ts b/completions-sample-code/vscode-node/lib/src/test/localFileSystem.test.ts new file mode 100644 index 0000000..6395441 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/localFileSystem.test.ts @@ -0,0 +1,99 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { mkdir, mkdtemp, rm, stat, symlink, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { FileType } from '../fileSystem'; +import { LocalFileSystem } from '../localFileSystem'; +import { makeFsUri } from '../util/uri'; + +suite('LocalFileSystem', function () { + let testDir: string; + const defaultFileSystem = new LocalFileSystem(); + + // only do all the file system work once for the suite + suiteSetup(async function () { + testDir = await mkdtemp(join(tmpdir(), 'copilot-unit-test-')); + await mkdir(join(testDir, 'folder')); + await symlink(join(testDir, 'folder'), join(testDir, 'folder-link'), 'dir'); + + await writeFile(join(testDir, 'file'), '\n'); + await symlink(join(testDir, 'file'), join(testDir, 'file-link')); + + await writeFile(join(testDir, 'tempfile'), ''); + await symlink(join(testDir, 'tempfile'), join(testDir, 'dangling-link')); + await rm(join(testDir, 'tempfile')); // leave the link dangling + }); + + suiteTeardown(async function () { + await rm(testDir, { recursive: true }); + }); + + test('.readDirectory returns correct entries', async function () { + const result = await defaultFileSystem.readDirectory(makeFsUri(testDir)); + assert.strictEqual(result.length, 5); + const target = [ + ['folder', FileType.Directory], + ['folder-link', FileType.Directory | FileType.SymbolicLink], + ['file', FileType.File], + ['file-link', FileType.File | FileType.SymbolicLink], + ['dangling-link', FileType.Unknown], + ]; + for (const entry of target) { + assert.ok( + result.some(([name, type]) => name === entry[0] && type === entry[1]), + `Expected entry ${entry[0]} with type ${entry[1]} not found in result` + ); + } + }); + + test('.stat returns correct stats for a normal file', async function () { + const fsStats = await stat(join(testDir, 'file')); + const result = await defaultFileSystem.stat(makeFsUri(join(testDir, 'file'))); + + assert.strictEqual(result.ctime, fsStats.ctimeMs); + assert.strictEqual(result.mtime, fsStats.mtimeMs); + assert.strictEqual(result.size, fsStats.size); + assert.strictEqual(result.type, FileType.File); + }); + + test('.stat returns correct stats for a directory', async function () { + const fsStats = await stat(join(testDir, 'folder')); + const result = await defaultFileSystem.stat(makeFsUri(join(testDir, 'folder'))); + + assert.strictEqual(result.ctime, fsStats.ctimeMs); + assert.strictEqual(result.mtime, fsStats.mtimeMs); + assert.strictEqual(result.size, fsStats.size); + assert.strictEqual(result.type, FileType.Directory); + }); + + test('.stat returns target stats and combined type for link to file', async function () { + const fsStats = await stat(join(testDir, 'file')); + const result = await defaultFileSystem.stat(makeFsUri(join(testDir, 'file-link'))); + + assert.strictEqual(result.ctime, fsStats.ctimeMs); + assert.strictEqual(result.mtime, fsStats.mtimeMs); + assert.strictEqual(result.size, fsStats.size); + assert.strictEqual(result.type, FileType.File | FileType.SymbolicLink); + }); + + test('.stat returns target stats and combined type for link to directory', async function () { + const fsStats = await stat(join(testDir, 'folder')); + const result = await defaultFileSystem.stat(makeFsUri(join(testDir, 'folder-link'))); + + assert.strictEqual(result.ctime, fsStats.ctimeMs); + assert.strictEqual(result.mtime, fsStats.mtimeMs); + assert.strictEqual(result.size, fsStats.size); + assert.strictEqual(result.type, FileType.Directory | FileType.SymbolicLink); + }); + + test('.stat returns Unknown type for a dangling link', async function () { + const result = await defaultFileSystem.stat(makeFsUri(join(testDir, 'dangling-link'))); + + assert.strictEqual(result.type, FileType.Unknown); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/loggerHelpers.ts b/completions-sample-code/vscode-node/lib/src/test/loggerHelpers.ts new file mode 100644 index 0000000..f388693 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/loggerHelpers.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as util from 'node:util'; +import { ICompletionsLogTargetService, LogLevel } from '../logger'; + +export type TestLogMessage = { + level: LogLevel; + category: string; + extra: unknown[]; +}; + +export class TestLogTarget implements ICompletionsLogTargetService { + declare _serviceBrand: undefined; + private readonly _messages: TestLogMessage[] = []; + + logIt(level: LogLevel, category: string, ...extra: unknown[]): void { + this._messages.push({ level, category: category, extra }); + } + + hasMessage(level: LogLevel, ...extra: unknown[]) { + return this._messages.some( + m => + m.level === level && + m.extra.length === extra.length && + m.extra + .filter(e => !(e instanceof Error)) + .every((e, i) => { + return util.isDeepStrictEqual(e, extra[i]); + }) + ); + } + + assertHasMessage(level: LogLevel, ...extra: unknown[]) { + if (!this.hasMessage(level, ...extra)) { + throw new Error( + `Expected message not found: ${LogLevel[level]} ${JSON.stringify( + extra + )}. Actual messages: ${this._messages + .map(m => '\n- ' + LogLevel[m.level] + ': ' + JSON.stringify(m.extra)) + .join('')}` + ); + } + } + + /** + * Checks for a logged message matching a given regex. Emulates + * OutputChannelLog for conversion of log message to string. + */ + hasMessageMatching(level: LogLevel, test: RegExp) { + return this._messages.some( + m => m.level === level && test.test(`[${m.category}] ${m.extra.map(toPlainText).join(',')}`) + ); + } + + assertHasMessageMatching(level: LogLevel, test: RegExp) { + if (!this.hasMessageMatching(level, test)) { + throw new Error( + `Expected message not found: ${LogLevel[level]} ${test}. Actual messages: ${this._messages + .map(m => '\n- ' + LogLevel[m.level] + ': ' + JSON.stringify(m.extra)) + .join('')}` + ); + } + } + + get messageCount() { + return this._messages.length; + } + + isEmpty() { + return this._messages.length === 0; + } +} + +function toPlainText(x: unknown): string { + switch (typeof x) { + case 'object': + return util.inspect(x); + default: + return String(x); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/networking.test.ts b/completions-sample-code/vscode-node/lib/src/test/networking.test.ts new file mode 100644 index 0000000..28e9966 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/networking.test.ts @@ -0,0 +1,83 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; + +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { + ICompletionsFetcherService, + postRequest +} from '../networking'; +import { createLibTestingContext } from './context'; +import { StaticFetcher, createFakeJsonResponse } from './fetcher'; + +suite('Networking test Suite', function () { + let accessor: ServicesAccessor; + let fetcher: StaticFetcher; + + setup(function () { + const serviceCollection = createLibTestingContext(); + fetcher = new StaticFetcher(); + serviceCollection.define(ICompletionsFetcherService, fetcher); + accessor = serviceCollection.createTestingAccessor(); + }); + + test('each request contains editor info headers', async function () { + await postRequest(accessor, 'http://localhost:8080/', '', undefined, 'id'); + + assert.strictEqual(fetcher.headerBuffer!['VScode-SessionId'], 'test-session'); + assert.strictEqual(fetcher.headerBuffer!['VScode-MachineId'], 'test-machine'); + assert.strictEqual(fetcher.headerBuffer!['Editor-Version'], 'lib-tests-editor/1'); + assert.strictEqual(fetcher.headerBuffer!['Editor-Plugin-Version'], 'lib-tests-plugin/2'); + assert.match(fetcher.headerBuffer!['Copilot-Language-Server-Version'], /^\d+\.\d+\./); + }); + + test('additional headers can be specified per-request', async function () { + await postRequest(accessor, 'http://localhost:8080/', '', undefined, 'id', undefined, undefined, { + 'X-Custom-Model': 'disable', + }); + + assert.strictEqual(fetcher.headerBuffer!['X-Custom-Model'], 'disable'); + }); + + suite('JSON Parsing', function () { + async function getJsonError(json: string, headers?: { [key: string]: string }): Promise<Error | undefined> { + try { + await createFakeJsonResponse(200, json, headers).json(); + } catch (e) { + if (e instanceof Error) { + return e; + } + throw e; + } + } + + test('parses valid JSON', async function () { + assert.deepStrictEqual(await createFakeJsonResponse(200, '{"a":"b"}').json(), { a: 'b' }); + }); + + test('throws an error for an unexpected content type', async function () { + const error = (await getJsonError('<!doctype>', { 'content-type': 'text/html' })) as NodeJS.ErrnoException; + assert.ok(error instanceof SyntaxError); + assert.deepStrictEqual(error.name, 'SyntaxError'); + }); + + test('throws an error for truncated JSON', async function () { + for (const json of ['{', '{"', '{"a"', '{"a":', '{"a":1', '{"a":1,']) { + const error = (await getJsonError(json)) as NodeJS.ErrnoException; + assert.ok(error instanceof SyntaxError); + assert.deepStrictEqual(error.name, 'SyntaxError'); + } + const error = (await getJsonError('{', { 'content-length': '2' })) as NodeJS.ErrnoException; + assert.ok(error instanceof SyntaxError); + assert.deepStrictEqual(error.name, 'SyntaxError'); + }); + + test('throws an error for any other parse failure', async function () { + const error = await getJsonError('&'); + assert.ok(error instanceof SyntaxError); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/noopTelemetry.ts b/completions-sample-code/vscode-node/lib/src/test/noopTelemetry.ts new file mode 100644 index 0000000..f420e56 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/noopTelemetry.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotTelemetryReporter } from '../telemetry'; + +export class NoopCopilotTelemetryReporter implements CopilotTelemetryReporter { + sendTelemetryEvent(): void { + // noop + } + sendTelemetryErrorEvent(): void { + // noop + } + dispose(): Promise<void> { + return Promise.resolve(); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/notificationSender.test.ts b/completions-sample-code/vscode-node/lib/src/test/notificationSender.test.ts new file mode 100644 index 0000000..ba10b85 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/notificationSender.test.ts @@ -0,0 +1,67 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { TestNotificationSender } from './testHelpers'; + +suite('NotificationSender test suite', function () { + test('should show information message every time when called without ID', async function () { + const notificationSender = new TestNotificationSender(); + const message = 'Operation completed successfully'; + await notificationSender.showInformationMessage(message); + await notificationSender.showInformationMessage(message); + const count = notificationSender.sentMessages.length; + assert.strictEqual( + count, + 2, + `Expected showInformationMessage to be called twice, but was called ${count} times` + ); + }); + + test('should return action when provided to information message', async function () { + const notificationSender = new TestNotificationSender(); + const action = { title: 'OK' }; + notificationSender.performAction('OK'); + + const result = await notificationSender.showInformationMessage('Success', action); + assert.deepStrictEqual(result, action); + }); + + test('should return undefined when action is dismissed for information message', async function () { + const notificationSender = new TestNotificationSender(); + notificationSender.performDismiss(); + + const result = await notificationSender.showInformationMessage('Success', { title: 'OK' }); + assert.strictEqual(result, undefined); + }); + + test('should show request message and return action', async function () { + const notificationSender = new TestNotificationSender(); + const action = { title: 'Yes' }; + notificationSender.performAction('Yes'); + + const result = await notificationSender.showInformationModal('Are you sure?', action, { title: 'No' }); + assert.deepStrictEqual(result, action); + assert.strictEqual(notificationSender.sentMessages.length, 1); + assert.strictEqual(notificationSender.sentMessages[0], 'Are you sure?'); + }); + + test('should return undefined when request is dismissed', async function () { + const notificationSender = new TestNotificationSender(); + notificationSender.performDismiss(); + + const result = await notificationSender.showInformationModal('Are you sure?', { title: 'Yes' }); + assert.strictEqual(result, undefined); + }); + + test('should handle request without actions', async function () { + const notificationSender = new TestNotificationSender(); + + const result = await notificationSender.showInformationModal('Just showing info'); + assert.strictEqual(result, undefined); + assert.strictEqual(notificationSender.sentMessages.length, 1); + assert.strictEqual(notificationSender.sentMessages[0], 'Just showing info'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/postInsertion.test.ts b/completions-sample-code/vscode-node/lib/src/test/postInsertion.test.ts new file mode 100644 index 0000000..2f6579b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/postInsertion.test.ts @@ -0,0 +1,171 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import Sinon from 'sinon'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsCitationManager, IPDocumentCitation } from '../citationManager'; +import { CopilotCompletion } from '../ghostText/copilotCompletion'; +import { ResultType } from '../ghostText/ghostText'; +import { postInsertionTasks } from '../postInsertion'; +import { TelemetryWithExp } from '../telemetry'; +import { IPosition, ITextDocument } from '../textDocument'; +import { ICompletionsTextDocumentManagerService } from '../textDocumentManager'; +import { ICompletionsPromiseQueueService } from '../util/promiseQueue'; +import { createLibTestingContext } from './context'; +import { fakeCodeReference } from './fetcher'; +import { TestTextDocumentManager } from './textDocument'; + +suite('postInsertionTasks', function () { + let accessor: ServicesAccessor; + let handleIPCodeCitation: Sinon.SinonSpy<[citation: IPDocumentCitation], Promise<void>>; + let docMgr: TestTextDocumentManager; + let doc: ITextDocument; + const uri = 'file:///hello.js'; + const pos: IPosition = { line: 1, character: 0 }; + const completionText = 'console.log("Hello, world!")'; + let completion: CopilotCompletion; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + const citationManager = accessor.get(ICompletionsCitationManager); + handleIPCodeCitation = Sinon.spy(citationManager, 'handleIPCodeCitation'); + docMgr = accessor.get(ICompletionsTextDocumentManagerService) as TestTextDocumentManager; + doc = docMgr.setTextDocument(uri, 'javascript', 'function main() {\n\n\n}'); + completion = { + uuid: '1234-5678-9abc', + insertText: completionText, + range: { start: pos, end: pos }, + uri: doc.uri, + telemetry: TelemetryWithExp.createEmptyConfigForTesting(), + displayText: 'console.log("Hello, world!")', + position: pos, + offset: doc.offsetAt(pos), + index: 0, + resultType: ResultType.Network, + clientCompletionId: '1234-5678-9abc', + }; + }); + + test('invokes CitationManager when code references are present in the completion', async function () { + completion.copilotAnnotations = fakeCodeReference(0, completionText.length); + const citations = ( + completion.copilotAnnotations.ip_code_citations[0].details as { citations: { license: string; url: string }[] } + ).citations; + + docMgr.updateTextDocument(doc.uri, `function main() {\n${completionText}\n\n}`); + postInsertionTasks( + accessor, + 'ghostText', + completionText, + completion.offset, + doc.uri, + completion.telemetry, + { compType: 'full', acceptedLength: completionText.length, acceptedLines: 0 }, + completion.copilotAnnotations + ); + const promiseQueue = accessor.get(ICompletionsPromiseQueueService); + await promiseQueue.flush(); + + Sinon.assert.calledOnceWithExactly(handleIPCodeCitation, { + inDocumentUri: doc.uri, + offsetStart: completion.offset, + offsetEnd: completion.offset + completionText.length, + version: doc.version + 1, + location: { start: pos, end: { line: pos.line, character: completionText.length } }, + matchingText: completionText, + details: citations, + }); + }); + + test('adjusts code reference offsets for partial acceptance', async function () { + completion.copilotAnnotations = fakeCodeReference(0, completionText.length); + const citations = ( + completion.copilotAnnotations.ip_code_citations[0].details as { citations: { license: string; url: string }[] } + ).citations; + const partial = completionText.slice(0, 11); + + docMgr.updateTextDocument(doc.uri, `function main() {\n${partial}\n\n}`); + postInsertionTasks( + accessor, + 'ghostText', + completionText, + completion.offset, + doc.uri, + completion.telemetry, + { compType: 'partial', acceptedLength: partial.length, acceptedLines: 0 }, + completion.copilotAnnotations + ); + const promiseQueue = accessor.get(ICompletionsPromiseQueueService); + await promiseQueue.flush(); + + Sinon.assert.calledOnceWithExactly(handleIPCodeCitation, { + inDocumentUri: doc.uri, + offsetStart: completion.offset, + offsetEnd: completion.offset + partial.length, + version: doc.version + 1, + location: { start: pos, end: { line: pos.line, character: partial.length } }, + matchingText: partial, + details: citations, + }); + }); + + test('does not invoke CitationManager when partially accepted completion excludes matched code', async function () { + completion.copilotAnnotations = fakeCodeReference(12, 14); // "Hello, world!" + const partial = completionText.slice(0, 11); + + docMgr.updateTextDocument(doc.uri, `function main() {\n${partial}\n\n}`); + postInsertionTasks( + accessor, + 'ghostText', + completionText, + completion.offset, + doc.uri, + completion.telemetry, + { compType: 'partial', acceptedLength: partial.length, acceptedLines: 0 }, + completion.copilotAnnotations + ); + const promiseQueue = accessor.get(ICompletionsPromiseQueueService); + await promiseQueue.flush(); + + Sinon.assert.notCalled(handleIPCodeCitation); + }); + + test('adjusts code reference range when additional document edits have been made since completion insertion', async function () { + completion.copilotAnnotations = fakeCodeReference(0, completionText.length); + const citations = ( + completion.copilotAnnotations.ip_code_citations[0].details as { citations: { license: string; url: string }[] } + ).citations; + + // when we'd like the editor to notify us of acceptance: + // docMgr.updateTextDocument(doc.uri, `function main() {\n${completionText}\n\n}`); + // when it might: + docMgr.updateTextDocument(doc.uri, `function main() {\n ${completionText};\n\n}`); + postInsertionTasks( + accessor, + 'ghostText', + completionText, + completion.offset, + doc.uri, + completion.telemetry, + { compType: 'full', acceptedLength: completionText.length, acceptedLines: 3 }, + completion.copilotAnnotations + ); + const promiseQueue = accessor.get(ICompletionsPromiseQueueService); + await promiseQueue.flush(); + + Sinon.assert.calledOnceWithExactly(handleIPCodeCitation, { + inDocumentUri: doc.uri, + offsetStart: completion.offset + 4, + offsetEnd: completion.offset + 4 + completionText.length, + version: doc.version + 1, + location: { + start: { line: pos.line, character: 4 }, + end: { line: pos.line, character: 4 + completionText.length }, + }, + matchingText: completionText, + details: citations, + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/runtimeMode.test.ts b/completions-sample-code/vscode-node/lib/src/test/runtimeMode.test.ts new file mode 100644 index 0000000..8137398 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/runtimeMode.test.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { + RuntimeMode +} from '../util/runtimeMode'; + +suite('RuntimeMode', function () { + suite('environment variable precedence', function () { + test('looks for a GH_COPILOT_ variable', function () { + const runtime = RuntimeMode.fromEnvironment(false, [], { GH_COPILOT_DEBUG: '1' }); + assert.strictEqual(runtime.flags.debug, true); + }); + + test('looks for a GITHUB_COPILOT_ variable', function () { + const runtime = RuntimeMode.fromEnvironment(false, [], { GITHUB_COPILOT_DEBUG: '1' }); + assert.strictEqual(runtime.flags.debug, true); + }); + + test('gives precedence to GH_COPILOT_ variable if both are set', function () { + const runtime = RuntimeMode.fromEnvironment(false, [], { + GH_COPILOT_DEBUG: '0', + GITHUB_COPILOT_DEBUG: '1', + }); + + assert.strictEqual(runtime.flags.debug, false); + }); + }); + + [true, false].forEach(inTest => { + test(`isRunningInTest is set to ${inTest}`, function () { + assert.strictEqual(RuntimeMode.fromEnvironment(inTest).isRunningInTest(), inTest); + }); + }); + + test('shouldFailForDebugPurposes is enabled by isRunningInTest', function () { + assert.strictEqual(RuntimeMode.fromEnvironment(true).shouldFailForDebugPurposes(), true); + }); + + suite('isVerboseLoggingEnabled', function () { + [ + 'GH_COPILOT_DEBUG', + 'GITHUB_COPILOT_DEBUG', + 'GH_COPILOT_VERBOSE', + 'GITHUB_COPILOT_VERBOSE', + 'COPILOT_AGENT_VERBOSE', + ].forEach(key => { + ['1', 'true', 'TRUE'].forEach(value => { + test(`is enabled by ${key}=${value}`, function () { + assert.strictEqual(RuntimeMode.fromEnvironment(false, [], { [key]: value }).isVerboseLoggingEnabled(), true); + }); + }); + }); + + test('is enabled by --debug flag', function () { + assert.strictEqual(RuntimeMode.fromEnvironment(false, ['--debug'], {}).isVerboseLoggingEnabled(), true); + }); + + test('is disabled by default', function () { + assert.strictEqual(RuntimeMode.fromEnvironment(false, [], {}).isVerboseLoggingEnabled(), false); + }); + }); + + suite('isDebugEnabled', function () { + ['GH_COPILOT_DEBUG', 'GITHUB_COPILOT_DEBUG'].forEach(key => { + ['1', 'true', 'TRUE'].forEach(value => { + test(`is enabled by ${key}=${value}`, function () { + assert.strictEqual(RuntimeMode.fromEnvironment(false, [], { [key]: value }).isDebugEnabled(), true); + }); + }); + }); + + test('is enabled by --debug flag', function () { + assert.strictEqual(RuntimeMode.fromEnvironment(false, ['--debug'], {}).isDebugEnabled(), true); + }); + + test('is disabled by default', function () { + assert.strictEqual(RuntimeMode.fromEnvironment(false, [], {}).isDebugEnabled(), false); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/snapshot.ts b/completions-sample-code/vscode-node/lib/src/test/snapshot.ts new file mode 100644 index 0000000..2e17d54 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/snapshot.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PromptSnapshotNode } from '../../../prompt/src/components/components'; + +interface PathSegment { + name: string; + index: string | number; +} + +/** + * Queries a prompt snapshot tree to find a node value using a dot-notation path. + * + * @param snapshot - Root snapshot node to query + * @param path - Dot-separated path to target node. Supports: + * - Simple paths: "parent.child.grandchild" + * - Array indices: "parent.children[0].name" + * - Array keys: "parent.children['key'].name" + * - Wildcards: "parent.*.name" matches any child node + * - Node names can contain letters, numbers, and special chars except dots and brackets + * - The first child is selected if no index is provided, else use [*] for all children + * + * @returns Value of the matched node as string + * @throws {Error} If path is invalid or node cannot be found + */ +export function querySnapshot(snapshot: PromptSnapshotNode, path: string): string | PromptSnapshotNode[] { + const segments = path + .trim() + .split('.') + .map(s => s.trim()); + let current = snapshot; + for (const segment of segments) { + if (!current?.children?.length) { + throw new Error(`No children found at path segment '${segment}'. Path: ${path}`); + } + const { name, index } = parsePathSegment(segment); + validateNodeName(name, current, segment, path); + validateNodeChildrenLength(index, current.children, segment, path); + if (typeof index === 'number') { + current = current.children[index]; + } else if (index === '*') { + break; + } else { + const child = current.children.find(c => c.path.includes(index)); + if (!child) { + throw new Error(`No children with index '${index}' found at path segment '${segment}'. Path: ${path}`); + } + current = child; + } + } + if (!current?.value) { + return current.children || []; + } + return current.value; +} + +function parsePathSegment(segment: string): PathSegment { + const match = segment.match(/^([^[]+)(?:\[(\d+|\*|["'][\w-]+["'])\])?$/); + if (!match) { + throw new Error(`Invalid path segment: ${segment}`); + } + const stringIndex = match[2] ?? 0; + const index = isNaN(Number(stringIndex)) ? stringIndex : Number(stringIndex); + + return { + name: match[1], + index, + }; +} + +function validateNodeName(name: string, current: PromptSnapshotNode, segment: string, path: string) { + if (name !== '*' && name !== current.name) { + throw new Error( + `Name mismatch at segment '${segment}'. Expected '${current.name}' but got '${name}'. Path: ${path}` + ); + } +} + +function validateNodeChildrenLength( + index: number | string, + children: PromptSnapshotNode[], + segment: string, + path: string +) { + if (typeof index === 'number' && index >= children.length) { + throw new Error( + `Index out of bounds at segment '${segment}'. Maximum index is ${children.length - 1}. Path: ${path}` + ); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/telemetry.test.ts b/completions-sample-code/vscode-node/lib/src/test/telemetry.test.ts new file mode 100644 index 0000000..6b2d1fe --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/telemetry.test.ts @@ -0,0 +1,131 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import Sinon from 'sinon'; +import { ICompletionsTelemetryService } from '../../../bridge/src/completionsTelemetryServiceBridge'; +import { ICompletionsTelemetryReporters, telemetryCatch, TelemetryData, TelemetryStore } from '../telemetry'; +import { ICompletionsTelemetryUserConfigService } from '../telemetry/userConfig'; +import { ICompletionsPromiseQueueService } from '../util/promiseQueue'; +import { createLibTestingContext } from './context'; +import { NoopCopilotTelemetryReporter } from './noopTelemetry'; +import { withInMemoryTelemetry } from './telemetry'; + +suite('Telemetry unit tests', function () { + const accessor = createLibTestingContext().createTestingAccessor(); + let clock: Sinon.SinonFakeTimers; + + setup(function () { + clock = Sinon.useFakeTimers(); + }); + + teardown(function () { + clock.restore(); + }); + + test('Adds additional fields', async function () { + const telemetry = TelemetryData.createAndMarkAsIssued(); + + await telemetry.makeReadyForSending(accessor, TelemetryStore.Standard, 'SkipExp', 2000); + + assert.ok(telemetry.properties.copilot_build); + assert.ok(telemetry.properties.copilot_buildType); + // assert.ok(telemetry.properties.copilot_trackingId); + assert.ok(telemetry.properties.editor_version); + assert.ok(telemetry.properties.editor_plugin_version); + assert.ok(telemetry.properties.client_machineid); + assert.ok(telemetry.properties.client_sessionid); + assert.ok(telemetry.properties.copilot_version); + assert.ok(telemetry.properties.runtime_version); + assert.ok(telemetry.properties.common_extname); + assert.ok(telemetry.properties.common_extversion); + assert.ok(telemetry.properties.common_vscodeversion); + // assert.ok(telemetry.properties.proxy_enabled); + // assert.ok(telemetry.properties.proxy_auth); + // assert.ok(telemetry.properties.proxy_kerberos_spn); + // assert.ok(telemetry.properties.reject_unauthorized); + assert.ok(telemetry.properties.unique_id); + }); + + test('Telemetry user config has undefined tracking id', function () { + const accessor = createLibTestingContext().createTestingAccessor(); + const config = accessor.get(ICompletionsTelemetryUserConfigService); + + assert.strictEqual(config.trackingId, undefined); + }); + + test('Test for multiplexProperties with only short values', function () { + const properties = { + key1: 'short value', + key2: 'another short value', + }; + + const result = TelemetryData.multiplexProperties(properties); + + assert.deepEqual(result, properties); + }); + + test('Test for multiplexProperties with a long value', function () { + const longValue = 'a'.repeat(19000) + 'b'; + const properties = { + key1: longValue, + }; + + const result = TelemetryData.multiplexProperties(properties); + + assert.strictEqual(Object.keys(result).length, 3); + assert.strictEqual(result.key1.length, 8192); + assert.strictEqual(result.key1_02.length, 8192); + assert.strictEqual(result.key1_03.length, 19001 - 16384); + // The last character should be 'b' if we sliced correctly + assert.strictEqual(result.key1_03.slice(-1), 'b'); + }); + + test('telemetryCatch', async function () { + const { enhancedReporter } = await withInMemoryTelemetry(accessor, accessor => { + telemetryCatch( + accessor.get(ICompletionsTelemetryService), + accessor.get(ICompletionsPromiseQueueService), + () => { + throw new Error('boom!'); + }, + 'exceptionTest' + )(); + }); + + // Chat has no Telemetry Store. + + // const standardEvent = reporter.events[0]; + // assert.ok(standardEvent); + const enhancedEvent = enhancedReporter.events[0]; + assert.ok(enhancedEvent); + + // assert.deepStrictEqual(standardEvent.properties.message, 'boom!'); + + assert.deepStrictEqual(enhancedEvent.properties.message, 'boom!'); + + // assert.ok(standardEvent.properties.restricted_unique_id); + // assert.deepStrictEqual(enhancedEvent.properties.unique_id, standardEvent.properties.restricted_unique_id); + }); +}); + +suite('TelemetryReporters unit tests', function () { + test('deactivate is safe to call synchronously', async function () { + const accessor = createLibTestingContext().createTestingAccessor(); + const oldRepoter = new NoopCopilotTelemetryReporter(); + const oldRestrictedReporter = new NoopCopilotTelemetryReporter(); + const reporters = accessor.get(ICompletionsTelemetryReporters); + reporters.setReporter(oldRepoter); + reporters.setEnhancedReporter(oldRestrictedReporter); + + const asyncWork = reporters.deactivate(); + const updatedReporter = reporters.getReporter(accessor); // snapshot these before awaiting the result + const updatedEnhancedReporter = reporters.getEnhancedReporter(accessor); + await asyncWork; + + assert.strictEqual(updatedReporter, undefined); + assert.strictEqual(updatedEnhancedReporter, undefined); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/telemetry.ts b/completions-sample-code/vscode-node/lib/src/test/telemetry.ts new file mode 100644 index 0000000..b2eedee --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/telemetry.ts @@ -0,0 +1,126 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { ICompletionsTelemetryService } from '../../../bridge/src/completionsTelemetryServiceBridge'; +import { ICompletionsTelemetryReporters } from '../telemetry'; +import { ICompletionsPromiseQueueService, PromiseQueue } from '../util/promiseQueue'; +import { TelemetrySpy } from './telemetrySpy'; + +export type EventData = { + baseType: 'EventData'; + baseData: { + ver: number; + name: string; + properties: { + copilot_build: string; + common_os: string; + [key: string]: string; + }; + measurements: { + timeSinceIssuedMs: number; + [key: string]: number; + }; + }; +}; + +export type ExceptionData = { + baseType: 'ExceptionData'; + baseData: { + ver: number; + exceptions: [ + { + hasFullStack: boolean; + parsedStack: [ + { + sizeInBytes: number; + level: number; + method: string; + assembly: string; + fileName: string; + line: number; + }?, + ]; + message: string; + typeName: string; + }, + ]; + properties: { + copilot_build: string; + common_os: string; + [key: string]: string; + }; + measurements: { + timeSinceIssuedMs: number; + [key: string]: number; + }; + severityLevel: number; + }; +}; + +export type CapturedTelemetry<Event = Record<string, unknown>> = { + ver: number; + sampleRate: number; + tags: { [key: string]: string }; + data: Event; + iKey: string; + name: string; + time: string; +}; + +export type AuthorizationHeader = string | undefined; + +export class TestPromiseQueue extends PromiseQueue { + async awaitPromises() { + // Distinct from flush() in that errors are thrown + await Promise.all(this.promises); + } +} + +// export function isStandardTelemetryMessage(message: CapturedTelemetry<unknown>): boolean { +// return message.iKey === APP_INSIGHTS_KEY; +// } + +// export function isEnhancedTelemetryMessage(message: CapturedTelemetry<unknown>): boolean { +// return message.iKey === APP_INSIGHTS_KEY_SECURE; +// } + +export function isEvent(message: CapturedTelemetry): message is CapturedTelemetry<EventData> { + return message.data.baseType === 'EventData'; +} + +export function isException(message: CapturedTelemetry): message is CapturedTelemetry<ExceptionData> { + return message.data.baseType === 'ExceptionData'; +} + +export function allEvents(messages: CapturedTelemetry[]): messages is CapturedTelemetry<EventData>[] { + for (const message of messages) { + if (!isEvent(message)) { + return false; + } + } + return true; +} + +export async function withInMemoryTelemetry<T>( + accessor: ServicesAccessor, + work: (accessor: ServicesAccessor) => T | Promise<T> +): Promise<{ reporter: TelemetrySpy; enhancedReporter: TelemetrySpy; result: T }> { + const reporter = new TelemetrySpy(); + const enhancedReporter = new TelemetrySpy(); + const telemetryService = accessor.get(ICompletionsTelemetryService); + const reporters = accessor.get(ICompletionsTelemetryReporters); + try { + telemetryService.setSpyReporters(reporter, enhancedReporter); + reporters.setReporter(reporter); + reporters.setEnhancedReporter(enhancedReporter); + const result = await work(accessor); + const queue = accessor.get(ICompletionsPromiseQueueService) as TestPromiseQueue; + await queue.awaitPromises(); + + return { reporter, enhancedReporter: enhancedReporter, result }; + } finally { + telemetryService.clearSpyReporters(); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/telemetrySpy.ts b/completions-sample-code/vscode-node/lib/src/test/telemetrySpy.ts new file mode 100644 index 0000000..7d83059 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/telemetrySpy.ts @@ -0,0 +1,125 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CopilotTelemetryReporter } from '../telemetry'; +import * as assert from 'assert'; + +type ReportedEvent = { name: string; properties: { [key: string]: string }; measurements: { [key: string]: number } }; +type ReportedError = { + name: string; + properties: { [key: string]: string }; + measurements: { [key: string]: number }; + errorProps?: string[]; +}; + +export class TelemetrySpy implements CopilotTelemetryReporter { + readonly events: ReportedEvent[] = []; + readonly errors: ReportedError[] = []; + + sendTelemetryEvent( + eventName: string, + properties: { + [key: string]: string; + } = {}, + measurements: { + [key: string]: number; + } = {} + ): void { + this.events.push({ + name: eventName, + properties, + measurements, + }); + } + + sendTelemetryErrorEvent( + eventName: string, + properties: { + [key: string]: string; + } = {}, + measurements: { + [key: string]: number; + } = {}, + errorProps?: string[] + ): void { + this.errors.push({ + name: eventName, + properties, + measurements, + errorProps, + }); + } + + sendTelemetryException( + error: Error, + properties: { + [key: string]: string; + } = {}, + measurements: { + [key: string]: number; + } = {} + ): void { + this.events.push({ + name: 'error.exception', + properties: { message: error.message, ...properties }, + measurements, + }); + } + + dispose(): Promise<void> { + return Promise.resolve(); + } + + get hasEvent(): boolean { + return this.events.length > 0; + } + + get hasError(): boolean { + return this.errors.length > 0; + } + + get exceptions(): ReportedEvent[] { + return this.events.filter(e => e.name === 'error.exception'); + } + + get hasException(): boolean { + return this.exceptions.length > 0; + } + + get firstEvent(): ReportedEvent | undefined { + return this.events[0]; + } + + get firstError(): ReportedError | undefined { + return this.errors[0]; + } + + get firstException(): ReportedEvent | undefined { + return this.exceptions[0]; + } + + eventsMatching(filter: (event: ReportedEvent) => boolean): ReportedEvent[] { + return this.events.filter(filter); + } + + eventByName(name: string): ReportedEvent { + const candidates = this.events.filter(e => e.name === name); + assert.strictEqual(candidates.length, 1, `Expected exactly one event with name ${name}`); + return candidates[0]; + } + + errorsMatching(filter: (event: ReportedError) => boolean): ReportedError[] { + return this.errors.filter(filter); + } + + exceptionsMatching(filter: (event: ReportedEvent) => boolean): ReportedEvent[] { + return this.exceptions.filter(filter); + } + + // equivalent of assertHasProperty in testing/telemetry.ts + assertHasProperty(assertion: (m: { [key: string]: string }) => boolean) { + assert.ok(this.eventsMatching(e => e.name !== 'ghostText.produced').every(e => assertion(e.properties))); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/testContentExclusion.ts b/completions-sample-code/vscode-node/lib/src/test/testContentExclusion.ts new file mode 100644 index 0000000..99c5634 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/testContentExclusion.ts @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IIgnoreService } from '../../../../../../platform/ignore/common/ignoreService'; +import { CancellationToken } from '../../../../../../util/vs/base/common/cancellation'; +import { URI } from '../../../../../../util/vs/base/common/uri'; + +export class MockIgnoreService implements IIgnoreService { + declare _serviceBrand: undefined; + + isEnabled = true; + isRegexExclusionsEnabled = true; + dispose(): void { } + + init(): Promise<void> { + this._alwaysIgnore = true; + this.setBlockList = []; + return Promise.resolve(); + } + + isCopilotIgnored(file: URI, token?: CancellationToken): Promise<boolean> { + if (this._alwaysIgnore) { + return Promise.resolve(true); + } + if (this.setBlockList.includes(file.toString())) { + return Promise.resolve(true); + } + return Promise.resolve(false); + } + + asMinimatchPattern(): Promise<string | undefined> { + return Promise.resolve(undefined); + } + + private _alwaysIgnore = false; + setAlwaysIgnore() { + this._alwaysIgnore = true; + } + + private setBlockList: string[] = []; + setBlockListUris(uris: string[]) { + this.setBlockList = uris; + } +} \ No newline at end of file diff --git a/completions-sample-code/vscode-node/lib/src/test/testHelpers.ts b/completions-sample-code/vscode-node/lib/src/test/testHelpers.ts new file mode 100644 index 0000000..0707000 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/testHelpers.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ActionItem, ICompletionsNotificationSender } from '../notificationSender'; +import { IPosition, IRange } from '../textDocument'; + +export function positionToString(p: IPosition) { + return `${p.line}:${p.character}`; +} + +export function rangeToString(r: IRange) { + return `[${positionToString(r.start)}--${positionToString(r.end)}]`; +} + +export function restoreEnvAfterTest() { + const origEnv: typeof process.env = { ...process.env }; + teardown(function () { + // remove any keys that were added + for (const key of Object.keys(process.env)) { + if (!(key in origEnv)) { + delete process.env[key]; + } + } + + // restore the original values + for (const key of Object.keys(origEnv)) { + process.env[key] = origEnv[key]; + } + }); +} + +export class TestNotificationSender implements ICompletionsNotificationSender { + declare _serviceBrand: undefined; + + readonly sentMessages: string[] = []; + protected warningPromises: Promise<ActionItem | undefined>[] = []; + protected informationPromises: Promise<ActionItem | undefined>[] = []; + protected actionToPerform: string | undefined; + + performDismiss() { + this.actionToPerform = 'DISMISS'; + } + + performAction(title: string) { + this.actionToPerform = title; + } + + showWarningMessage(message: string, ...actions: ActionItem[]): Promise<ActionItem | undefined> { + this.sentMessages.push(message); + + let warningPromise: Promise<ActionItem | undefined>; + if (this.actionToPerform) { + if (this.actionToPerform === 'DISMISS') { + warningPromise = Promise.resolve(undefined); + } else { + const action = actions.find(a => a.title === this.actionToPerform); + warningPromise = action ? Promise.resolve(action) : Promise.resolve(undefined); + } + } else { + // If not set, default to the first action + warningPromise = actions ? Promise.resolve(actions[0]) : Promise.resolve(undefined); + } + + this.warningPromises.push(warningPromise); + return warningPromise; + } + + showInformationMessage(message: string, ...actions: ActionItem[]): Promise<ActionItem | undefined> { + this.sentMessages.push(message); + + let informationPromise: Promise<ActionItem | undefined>; + if (this.actionToPerform) { + if (this.actionToPerform === 'DISMISS') { + informationPromise = Promise.resolve(undefined); + } else { + const action = actions.find(a => a.title === this.actionToPerform); + informationPromise = action ? Promise.resolve(action) : Promise.resolve(undefined); + } + } else { + // If not set, default to the first action + informationPromise = actions ? Promise.resolve(actions[0]) : Promise.resolve(undefined); + } + + this.informationPromises.push(informationPromise); + return informationPromise; + } + + showInformationModal(message: string, ...actions: ActionItem[]): Promise<ActionItem | undefined> { + return this.showInformationMessage(message, ...actions); + } + + async waitForMessages() { + await Promise.all(this.warningPromises); + await Promise.all(this.informationPromises); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/textDocument.test.ts b/completions-sample-code/vscode-node/lib/src/test/textDocument.test.ts new file mode 100644 index 0000000..f4be0b0 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/textDocument.test.ts @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { createTextDocument } from './textDocument'; + +suite('TextDocument Tests', function () { + const newLineChars = ['\n', '\r\n', '\r']; + + for (const newLineChar of newLineChars) { + test(`new lines are handled correctly (${JSON.stringify(newLineChar)} separator)`, function () { + const doc = createTextDocument('file:///test.ts', 'typescript', 1, `hello${newLineChar}goodbye`); + + assert.deepStrictEqual(doc.lineCount, 2); + + const firstLine = doc.lineAt(0).text; + const lastLine = doc.lineAt(doc.lineCount - 1).text; + + assert.deepStrictEqual(firstLine, 'hello'); + assert.deepStrictEqual(lastLine, 'goodbye'); + }); + } +}); diff --git a/completions-sample-code/vscode-node/lib/src/test/textDocument.ts b/completions-sample-code/vscode-node/lib/src/test/textDocument.ts new file mode 100644 index 0000000..4ce8c8b --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/textDocument.ts @@ -0,0 +1,169 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { WorkspaceFolder } from '../../../types/src'; +import { CopilotTextDocument, INotebookCell, INotebookDocument, ITextDocument } from '../textDocument'; +import { + TextDocumentChangeEvent, + TextDocumentCloseEvent, + TextDocumentFocusedEvent, + TextDocumentManager, + TextDocumentOpenEvent, + WorkspaceFoldersChangeEvent, +} from '../textDocumentManager'; +import { Emitter } from '../util/event'; +import { basename, validateUri } from '../util/uri'; + +export function createTextDocument( + uri: string, + clientAndDetectedLanguageId: string, + version: number, + text: string +): ITextDocument { + return CopilotTextDocument.create( + validateUri(uri), + clientAndDetectedLanguageId, + version, + text, + clientAndDetectedLanguageId + ); +} + +interface JupyterCellVSCodeMetadata { + languageId?: string; +} + +interface JupyterCellMetadata { + vscode?: JupyterCellVSCodeMetadata; + [key: string]: unknown; +} + +interface JupyterCell { + cell_type: 'code' | 'markdown'; + source: string[]; + metadata: JupyterCellMetadata; +} + +interface JupyterNotebook { + cells: JupyterCell[]; + metadata: Record<string, unknown>; + nbformat: number; + nbformat_minor: number; +} + +export function parseNotebook(doc: ITextDocument): INotebookDocument { + const notebook: JupyterNotebook = JSON.parse(doc.getText()) as JupyterNotebook; + const cells: INotebookCell[] = notebook.cells.map((cell, index) => { + const cellUri = `${doc.uri.replace(/#.*/, '')}#${index}`; + const cellText = Array.isArray(cell.source) ? cell.source.join('') : cell.source; + + const languageId = + (cell.metadata?.['vscode']?.['languageId'] as string) || + (cell.cell_type === 'code' ? 'python' : 'markdown'); + + const document = CopilotTextDocument.create(cellUri, languageId, 0, cellText, languageId); + + return { + index, + document, + metadata: cell.metadata, + kind: cell.cell_type === 'code' ? 2 : 1, + }; + }); + return new InMemoryNotebookDocument(cells); +} + +export class InMemoryNotebookDocument implements INotebookDocument { + constructor(private readonly _cells: INotebookCell[]) { } + getCells(): INotebookCell[] { + return this._cells; + } + getCellFor({ uri }: { uri: string }): INotebookCell | undefined { + return this._cells.find(cell => cell.document.uri === uri); + } +} + +/** + * A concrete implementation of TextDocumentManager intended for use with the FakeFileSystem. + */ +export class SimpleTestTextDocumentManager extends TextDocumentManager { + private _openTextDocuments: ITextDocument[] = []; + private _notebookDocuments: Map<string, INotebookDocument> = new Map(); + private _workspaceFolders: WorkspaceFolder[] = []; + + init(workspaceFolders: { readonly uri: string; readonly name?: string }[]) { + this._workspaceFolders = workspaceFolders.map(f => ({ uri: f.uri, name: f.name ?? basename(f.uri) })); + } + + // Make public to allow for stubbing + override async readTextDocumentFromDisk(uri: string): Promise<string | undefined> { + return super.readTextDocumentFromDisk(uri); + } + + override getTextDocumentsUnsafe(): ITextDocument[] { + return this._openTextDocuments; + } + + readonly didFocusTextDocumentEmitter = new Emitter<TextDocumentFocusedEvent>(); + onDidFocusTextDocument = this.didFocusTextDocumentEmitter.event; + + readonly didChangeTextDocumentEmitter = new Emitter<TextDocumentChangeEvent>(); + onDidChangeTextDocument = this.didChangeTextDocumentEmitter.event; + + readonly didOpenTextDocumentEmitter = new Emitter<TextDocumentOpenEvent>(); + onDidOpenTextDocument = this.didOpenTextDocumentEmitter.event; + + readonly didCloseTextDocumentEmitter = new Emitter<TextDocumentCloseEvent>(); + onDidCloseTextDocument = this.didCloseTextDocumentEmitter.event; + + readonly didChangeWorkspaceFoldersEmitter = new Emitter<WorkspaceFoldersChangeEvent>(); + onDidChangeWorkspaceFolders = this.didChangeWorkspaceFoldersEmitter.event; + + setTextDocument(uri: string, languageId: string, text: string): ITextDocument { + const doc = createTextDocument(uri, languageId, 0, text); + this._openTextDocuments.push(doc); + return doc; + } + + updateTextDocument(uri: string, newText: string) { + const idx = this._openTextDocuments.findIndex(t => t.uri === uri.toString()); + if (idx < 0) { + throw new Error('Document not found'); + } + + const oldDoc = this._openTextDocuments[idx]; + this._openTextDocuments[idx] = createTextDocument(uri, oldDoc.clientLanguageId, oldDoc.version + 1, newText); + } + + setNotebookDocument(doc: ITextDocument, notebook: INotebookDocument) { + // Document URIs in the same notebook differ only by fragment + this._notebookDocuments.set(doc.uri.replace(/#.*/, ''), notebook); + } + + findNotebook({ uri }: { uri: string }): INotebookDocument | undefined { + return this._notebookDocuments.get(uri.replace(/#.*/, '')); + } + + getWorkspaceFolders() { + return this._workspaceFolders; + } +} + +/** + * An implementation of TextDocumentManager that is limited to documents you + * provide it. It will not attempt to open documents from the file system, but + * you may provide it with "closed" documents available for opening. + */ +export class TestTextDocumentManager extends SimpleTestTextDocumentManager { + private contents = new Map<string, string>(); + + override readTextDocumentFromDisk(uri: string): Promise<string | undefined> { + return Promise.resolve(this.contents.get(uri)); + } + + setDiskContents(uri: string, text: string) { + this.contents.set(uri, text); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/test/textDocumentManager.test.ts b/completions-sample-code/vscode-node/lib/src/test/textDocumentManager.test.ts new file mode 100644 index 0000000..5a2f6ad --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/test/textDocumentManager.test.ts @@ -0,0 +1,103 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { makeFsUri } from '../util/uri'; +import { createLibTestingContext } from './context'; +import { SimpleTestTextDocumentManager, createTextDocument } from './textDocument'; + +suite('TextDocumentManager base class', () => { + let textDocumentManager: SimpleTestTextDocumentManager; + let accessor: ServicesAccessor; + + setup(function () { + accessor = createLibTestingContext().createTestingAccessor(); + textDocumentManager = accessor.get(IInstantiationService).createInstance(SimpleTestTextDocumentManager); + }); + + test('should return the relative path of the document without workspaces', () => { + const mockDocument = createTextDocument(makeFsUri('/path/to/file.txt'), '', 0, ''); + + const relativePath = textDocumentManager.getRelativePath(mockDocument); + + assert.strictEqual(relativePath, 'file.txt'); + }); + + test('should return the relative path of the document in workspace', () => { + textDocumentManager.init([{ uri: 'file:///path/to/workspace' }]); + const mockDocument = createTextDocument(makeFsUri('/path/to/workspace/folder/file.txt'), '', 0, ''); + + const relativePath = textDocumentManager.getRelativePath(mockDocument); + + assert.strictEqual(relativePath, 'folder/file.txt'); + }); + + test('should return the relative path of the document in workspace with trailing slash', () => { + textDocumentManager.init([{ uri: 'file:///path/to/workspace/' }]); + const mockDocument = createTextDocument(makeFsUri('/path/to/workspace/folder/file.txt'), '', 0, ''); + + const relativePath = textDocumentManager.getRelativePath(mockDocument); + + assert.strictEqual(relativePath, 'folder/file.txt'); + }); + + test('should return undefined for untitled documents', () => { + const mockDocument = createTextDocument('untitled:Untitled-1', '', 0, ''); + + const relativePath = textDocumentManager.getRelativePath(mockDocument); + + assert.strictEqual(relativePath, undefined); + }); + + test('.getTextDocumentUnsafe() returns an existing document', function () { + textDocumentManager.setTextDocument('file:///path/to/file.txt', 'plaintext', 'file content'); + + const result = textDocumentManager.getTextDocumentUnsafe({ uri: 'file:///path/to/file.txt' }); + + assert.ok(result); + assert.strictEqual(result?.getText(), 'file content'); + }); + + test('.getTextDocumentUnsafe() returns undefined for an unopened document', function () { + const result = textDocumentManager.getTextDocumentUnsafe({ uri: 'file:///path/to/file.txt' }); + + assert.strictEqual(result, undefined); + }); + + test('.getTextDocumentUnsafe() normalizes URIs', function () { + textDocumentManager.setTextDocument('file:///c%3A/file', 'plaintext', 'file content'); + + const result = textDocumentManager.getTextDocumentUnsafe({ uri: 'file:///C:/file' }); + + assert.ok(result); + assert.strictEqual(result?.getText(), 'file content'); + }); + + test('.getTextDocument() finds documents by normalized URI', async function () { + const saved = textDocumentManager.setTextDocument('file:///c%3A/file', 'plaintext', 'file content'); + + const retrieved = await textDocumentManager.getTextDocument({ uri: 'file:///C:/file' }); + + assert.strictEqual(retrieved, saved); + }); + + test('.getTextDocument() returns undefined for anything other than an open document', async function () { + const result = await textDocumentManager.getTextDocument({ uri: 'file:///path/to/file.txt' }); + + assert.strictEqual(result, undefined); + }); + + test('.getTextDocument() retrieves the document synchronously', async function () { + textDocumentManager.setTextDocument('file:///path/to/file.txt', 'plaintext', 'file content'); + + const thenable = textDocumentManager.getTextDocument({ uri: 'file:///path/to/file.txt' }); + textDocumentManager.updateTextDocument('file:///path/to/file.txt', 'new content'); + const document = await thenable; + + assert.strictEqual(document?.version, 0); + assert.strictEqual(document?.getText(), 'file content'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/textDocument.ts b/completions-sample-code/vscode-node/lib/src/textDocument.ts new file mode 100644 index 0000000..c4fd398 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/textDocument.ts @@ -0,0 +1,288 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { detectLanguage } from './language/languageDetection'; +import { normalizeUri } from './util/uri'; +import { TextEdit } from '../../types/src'; +import { TextDocumentContentChangeEvent } from 'vscode-languageserver-protocol'; +import { TextDocument as LspTextDocument } from 'vscode-languageserver-textdocument'; +import { Position, Range, SelectedCompletionInfo } from 'vscode-languageserver-types'; + +export { type Position as IPosition, type Range as IRange } from '../../types/src'; + +export class LocationFactory { + static range = Range.create.bind(Range); + static position = Position.create.bind(Position); +} + +interface Line { + /** + * The line's text content. Doesn't include the trailing newline + */ + text: string; + range: Range; + isEmptyOrWhitespace: boolean; +} + +export type IntelliSenseInsertion = SelectedCompletionInfo & { + /** + * The corresponding signature information found in the tooltip. + */ + tooltipSignature?: string; +}; + +/** + * Used to represent validation result of retrieving a text document + */ +export type TextDocumentValidation = + | { status: 'invalid'; reason: string } + | { status: 'notfound'; message: string } + | { status: 'valid' }; +/** + * Used to represent validation result of retrieving a text document, plus the document itself if valid + */ +export type TextDocumentResult<TD = TextDocumentContents> = + | { status: 'invalid'; reason: string } + | { status: 'notfound'; message: string } + | { status: 'valid'; document: TD }; + +export interface TextDocumentIdentifier { + readonly uri: string; +} + +export interface TextDocumentContents { + /** + * Normalized version of .clientUri. Intended to become identical to .clientUri in the future, for better + * vscode-languageserver interop. + * + * @readonly + */ + readonly uri: string; + + /** + * The identifier of the detected language associated with this document. + * + * @readonly + */ + readonly detectedLanguageId: string; + + /** + * Get the text of this document. A substring can be retrieved by + * providing a range. + * + * @param range (optional) An range within the document to return. + * If no range is passed, the full content is returned. + * Invalid range positions are adjusted as described in `Position.line` and `Position.character`. + * If the start range position is greater than the end range position, + * then the effect of getText is as if the two positions were swapped. + + * @return The text of this document or a substring of the text if a + * range is provided. + */ + getText(range?: Range): string; + + /** + * Converts a zero-based offset to a position. + * + * @param offset A zero-based offset. + * @return A valid `position`. + */ + positionAt(offset: number): Position; + + /** + * Converts the position to a zero-based offset. + * Invalid positions are adjusted as described in `Position.line` and `Position.character`. + * + * @param position A position. + * @return A valid zero-based offset. + */ + offsetAt(position: Position): number; + + /** + * The number of lines in this document. + * + * @readonly + */ + readonly lineCount: number; + + /** + * Returns a text line denoted by the line number. + */ + lineAt(position: number | Position): Line; + + /** + * Return a copy of a document with the same version number and edits both applied and reflected in .appliedEdits. + */ + applyEdits(edits: TextEdit[]): TextDocumentContents; +} + +export interface ITextDocument extends TextDocumentContents { + /** + * The original URI provided by the client. + * + * @readonly + */ + readonly clientUri: string; + + /** + * The client reported identifier of the language associated with this document. + * + * @readonly + * @deprecated Favor the explicitly named clientLanguageId or detectedLanguageId + */ + readonly languageId: string; + + /** + * The client reported identifier of the language associated with this document. + * + * @readonly + */ + readonly clientLanguageId: string; + + /** + * The version number of this document (it will increase after each + * change, including undo/redo). + * + * @readonly + */ + readonly version: number; + + /** + * Return a copy of a document with the same version number and edits both applied and reflected in .appliedEdits. + */ + applyEdits(edits: TextEdit[]): ITextDocument; +} + +export interface INotebookCell { + /** + * The index of this cell in its `NotebookDocument.cellAt` containing notebook. The + * index is updated when a cell is moved within its notebook. The index is `-1` + * when the cell has been removed from its notebook. + */ + readonly index: number; + + /** + * The text of this cell, represented as `ITextDocument`. + */ + readonly document: ITextDocument; + + /** + * The metadata of this cell. Can be anything but must be JSON-stringifyable. + */ + readonly metadata: { [key: string]: unknown }; + + /** + * The kind of this cell. + * 1 = Markup + * 2 = Code + */ + readonly kind: 1 | 2; +} + +export interface INotebookDocument { + /** + * Get the cells of this notebook. + * + * @returns The cells contained by the range or all cells. + */ + getCells(): INotebookCell[]; + + getCellFor({ uri }: { uri: string }): INotebookCell | undefined; +} + +export class CopilotTextDocument implements ITextDocument { + private constructor( + readonly uri: string, + private readonly _textDocument: LspTextDocument, + readonly detectedLanguageId: string + ) { } + + /** + * Return a copy of a document with a new version number and changes applied. Used when a document is changed + * canonically (e.g., synced via textDocument/didChange). + */ + static withChanges(textDocument: ITextDocument, changes: TextDocumentContentChangeEvent[], version: number) { + const lspDoc = LspTextDocument.create( + textDocument.clientUri, + textDocument.clientLanguageId, + version, + textDocument.getText() + ); + LspTextDocument.update(lspDoc, changes, version); + return new CopilotTextDocument(textDocument.uri, lspDoc, textDocument.detectedLanguageId); + } + + /** + * Return a copy of a document with the same version number and edits applied. + * Used when the changes *aren't* canonical (e.g., a speculative completion request). + */ + applyEdits(edits: TextEdit[]) { + const lspDoc = LspTextDocument.create(this.clientUri, this.clientLanguageId, this.version, this.getText()); + LspTextDocument.update( + lspDoc, + edits.map(c => ({ text: c.newText, range: c.range })), + this.version + ); + return new CopilotTextDocument(this.uri, lspDoc, this.detectedLanguageId); + } + + static create( + uri: string, + languageId: string, + version: number, + text: string, + detectedLanguageId = detectLanguage({ uri, languageId }) + ) { + return new CopilotTextDocument( + normalizeUri(uri), + LspTextDocument.create(uri, languageId, version, text), + detectedLanguageId + ); + } + + get clientUri(): string { + return this._textDocument.uri; + } + + get clientLanguageId(): string { + return this._textDocument.languageId; + } + + get languageId(): string { + return this._textDocument.languageId; + } + + get version(): number { + return this._textDocument.version; + } + + get lineCount() { + return this._textDocument.lineCount; + } + + getText(range?: Range): string { + return this._textDocument.getText(range); + } + + positionAt(offset: number): Position { + return this._textDocument.positionAt(offset); + } + + offsetAt(position: Position): number { + return this._textDocument.offsetAt(position); + } + + lineAt(position: number | Position) { + const lineNumber = typeof position === 'number' ? position : position.line; + if (lineNumber < 0 || lineNumber >= this.lineCount) { + throw new RangeError('Illegal value for lineNumber'); + } + const rangeWithNewline = Range.create(lineNumber, 0, lineNumber + 1, 0); + const text = this.getText(rangeWithNewline).replace(/\r\n$|\r$|\n$/g, ''); + const range = Range.create(Position.create(lineNumber, 0), Position.create(lineNumber, text.length)); + + const isEmptyOrWhitespace = text.trim().length === 0; + return { text, range, isEmptyOrWhitespace }; + } +} diff --git a/completions-sample-code/vscode-node/lib/src/textDocumentManager.ts b/completions-sample-code/vscode-node/lib/src/textDocumentManager.ts new file mode 100644 index 0000000..c0e1698 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/textDocumentManager.ts @@ -0,0 +1,282 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { createServiceIdentifier } from '../../../../../util/common/services'; +import { IInstantiationService } from '../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TextDocumentItem, VersionedTextDocumentIdentifier, WorkspaceFolder } from '../../types/src'; +import { ICompletionsFileSystemService } from './fileSystem'; +import { + INotebookDocument, + IRange, + ITextDocument, + TextDocumentIdentifier, + TextDocumentResult, + TextDocumentValidation, +} from './textDocument'; +import { isDocumentValid } from './util/documentEvaluation'; +import { Event } from './util/event'; +import { basename, normalizeUri } from './util/uri'; + +/** + * An interface describing an individual change in the text of a document. + */ +interface TextDocumentContentChangeEvent { + /** + * The range that got replaced. + */ + readonly range: IRange; + /** + * The offset of the range that got replaced. + */ + readonly rangeOffset: number; + /** + * The length of the range that got replaced. + */ + readonly rangeLength: number; + /** + * The new text for the range. + */ + readonly text: string; +} + +/** + * An event describing a document open. + * @public KEEPING FOR TESTS + */ +export interface TextDocumentOpenEvent { + /** + * The affected document. + */ + readonly document: TextDocumentItem; +} + +/** + * An event describing a transactional document change. + * @public KEEPING FOR TESTS + */ +export interface TextDocumentChangeEvent { + /** + * The affected document. + */ + readonly document: VersionedTextDocumentIdentifier; + + /** + * An array of content changes. + */ + readonly contentChanges: readonly TextDocumentContentChangeEvent[]; +} + +/** + * An event describing a document close. + * @public KEEPING FOR TESTS + */ +export interface TextDocumentCloseEvent { + readonly document: TextDocumentIdentifier; +} + +/** @public KEEPING FOR TESTS */ +export interface TextDocumentFocusedEvent { + readonly document?: TextDocumentIdentifier; +} + +export interface WorkspaceFoldersChangeEvent { + readonly workspaceFolders: WorkspaceFolder[]; + readonly added: WorkspaceFolder[]; + readonly removed: WorkspaceFolder[]; +} + +export const ICompletionsTextDocumentManagerService = createServiceIdentifier<ICompletionsTextDocumentManagerService>('ICompletionsTextDocumentManagerService'); + +export interface ICompletionsTextDocumentManagerService { + readonly _serviceBrand: undefined; + onDidChangeTextDocument: Event<TextDocumentChangeEvent>; + onDidOpenTextDocument: Event<TextDocumentOpenEvent>; + onDidCloseTextDocument: Event<TextDocumentCloseEvent>; + onDidFocusTextDocument: Event<TextDocumentFocusedEvent>; + onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>; + + textDocuments(): Promise<ITextDocument[]>; + + /** + * Get all open text documents, skipping content exclusions and other validations. + */ + getTextDocumentsUnsafe(): ITextDocument[]; + + /** + * Get the text document for the given URI, skipping content exclusions and other validations. + */ + getTextDocumentUnsafe(docId: TextDocumentIdentifier): ITextDocument | undefined; + + /** + * Get the text document for the given URI, checking content exclusions and other validations. + */ + getTextDocument(docId: TextDocumentIdentifier): Promise<ITextDocument | undefined>; + + /** + * Get a TextDocumentValidation for the given document URI. Unlike other methods, this supports reading the + * document from disk. + */ + getTextDocumentValidation(docId: TextDocumentIdentifier): Promise<TextDocumentValidation>; + + /** + * Get a TextDocumentResult for the given document URI. + */ + getTextDocumentWithValidation(docId: TextDocumentIdentifier): Promise<TextDocumentResult<ITextDocument>>; + + /** + * If `TextDocument` represents notebook returns `INotebookDocument` instance, otherwise returns `undefined` + */ + findNotebook(doc: TextDocumentIdentifier): INotebookDocument | undefined; + + getWorkspaceFolders(): WorkspaceFolder[]; + + getWorkspaceFolder(doc: TextDocumentIdentifier): WorkspaceFolder | undefined; + + /** + * Get the path of the given document relative to one of the workspace folders, + * or its basename if it is not under any of the workspace folders. + * Returns `undefined` if the file is untitled. + */ + getRelativePath(doc: TextDocumentIdentifier): string | undefined; +} + +export abstract class TextDocumentManager implements ICompletionsTextDocumentManagerService { + declare _serviceBrand: undefined; + abstract onDidChangeTextDocument: Event<TextDocumentChangeEvent>; + abstract onDidOpenTextDocument: Event<TextDocumentOpenEvent>; + abstract onDidCloseTextDocument: Event<TextDocumentCloseEvent>; + + abstract onDidFocusTextDocument: Event<TextDocumentFocusedEvent>; + abstract onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>; + + /** + * Get all open text documents, skipping content exclusions and other validations. + */ + abstract getTextDocumentsUnsafe(): ITextDocument[]; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @ICompletionsFileSystemService private readonly fileSystem: ICompletionsFileSystemService, + ) { } + + async textDocuments(): Promise<ITextDocument[]> { + const documents = this.getTextDocumentsUnsafe(); + const filteredDocuments: ITextDocument[] = []; + for (const doc of documents) { + const result = await this.instantiationService.invokeFunction(isDocumentValid, doc); + // Only return valid documents + if (result.status === 'valid') { + filteredDocuments.push(doc); + } + } + return filteredDocuments; + } + + /** + * Get the text document for the given URI, skipping content exclusions and other validations. + */ + getTextDocumentUnsafe(docId: TextDocumentIdentifier): ITextDocument | undefined { + const uri = normalizeUri(docId.uri); + return this.getTextDocumentsUnsafe().find(t => t.uri === uri); + } + + /** + * Get the text document for the given URI, checking content exclusions and other validations. + */ + async getTextDocument(docId: TextDocumentIdentifier): Promise<ITextDocument | undefined> { + return this.getTextDocumentWithValidation(docId).then(result => { + if (result.status === 'valid') { + return result.document; + } + return undefined; + }); + } + + private async validateTextDocument(docId: TextDocumentIdentifier) { + return await this.instantiationService.invokeFunction(isDocumentValid, docId); + } + + /** + * Get a TextDocumentValidation for the given document URI. Unlike other methods, this supports reading the + * document from disk. + */ + async getTextDocumentValidation(docId: TextDocumentIdentifier): Promise<TextDocumentValidation> { + try { + return await this.validateTextDocument(docId); + } catch (err) { + return this.notFoundResult(docId); + } + } + + /** + * Get a TextDocumentResult for the given document URI. + */ + async getTextDocumentWithValidation(docId: TextDocumentIdentifier): Promise<TextDocumentResult<ITextDocument>> { + const document = this.getTextDocumentUnsafe(docId); + if (!document) { return this.notFoundResult(docId); } + const result = await this.validateTextDocument(docId); + return result.status === 'valid' ? { status: 'valid', document } : result; + } + + private notFoundResult({ uri }: TextDocumentIdentifier): { status: 'notfound'; message: string } { + return { + status: 'notfound', + message: `Document for URI could not be found: ${uri}`, + }; + } + + /** + * Implements ability to open a text document that is currently not open (and not tracked by the document manager). + * + * This is usually used with asychronous operations like the postInsertion callbacks that + * analyze a document long time after the user interacted with it. + */ + protected async readTextDocumentFromDisk(uri: string): Promise<string | undefined> { + try { + const fileStat = await this.fileSystem.stat(uri); + if (fileStat.size > 5 * 1024 * 1024) { + return undefined; + } + } catch (e) { + // ignore if file does not exist + return undefined; + } + return await this.fileSystem.readFileString(uri); + } + + /** + * If `TextDocument` represents notebook returns `INotebookDocument` instance, otherwise returns `undefined` + */ + abstract findNotebook(doc: TextDocumentIdentifier): INotebookDocument | undefined; + + abstract getWorkspaceFolders(): WorkspaceFolder[]; + + getWorkspaceFolder(doc: TextDocumentIdentifier) { + const uri = normalizeUri(doc.uri); + return this.getWorkspaceFolders().find(f => uri.startsWith(normalizeUri(f.uri))); + } + + /** + * Get the path of the given document relative to one of the workspace folders, + * or its basename if it is not under any of the workspace folders. + * Returns `undefined` if the file is untitled. + */ + getRelativePath(doc: TextDocumentIdentifier): string | undefined { + if (doc.uri.startsWith('untitled:')) { + // matches the internal implementation of .isUntitled on vscode.TextDocument + // and example URLs in the LSP spec + return undefined; + } + const uri = normalizeUri(doc.uri); + for (const folder of this.getWorkspaceFolders()) { + const parentURI = normalizeUri(folder.uri) + .replace(/[#?].*/, '') + .replace(/\/?$/, '/'); + if (uri.startsWith(parentURI)) { + return uri.slice(parentURI.length); + } + } + return basename(uri); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/util/async.ts b/completions-sample-code/vscode-node/lib/src/util/async.ts new file mode 100644 index 0000000..cfa45f3 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/async.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Deferred promise implementation to enable delayed promise resolution. + * Note: in Node 22+ this can be replaced with Promise.withResolvers. + */ +export class Deferred<T> { + resolve: (value: T | PromiseLike<T>) => void = () => { }; + reject: (reason?: unknown) => void = () => { }; + + readonly promise: Promise<T> = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); +} + +/** + * Returns a promise that resolves after a delay, optionally with a value. + * Equivalent to node:timers/promises setTimeout without node dependency. + */ +export function delay<T>(ms: number, value: T): Promise<T>; +export function delay(ms: number): Promise<void>; +export function delay(ms: number, value = undefined) { + return new Promise(resolve => setTimeout(() => resolve(value), ms)); +} diff --git a/completions-sample-code/vscode-node/lib/src/util/documentEvaluation.ts b/completions-sample-code/vscode-node/lib/src/util/documentEvaluation.ts new file mode 100644 index 0000000..54e39da --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/documentEvaluation.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IIgnoreService } from '../../../../../../platform/ignore/common/ignoreService'; +import { URI } from '../../../../../../util/vs/base/common/uri'; +import { ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { TextDocumentIdentifier } from '../textDocument'; + +/** + * Evaluate document uri to see if it's valid for copilot to process + */ +export async function isDocumentValid( + accessor: ServicesAccessor, + document: TextDocumentIdentifier, +): Promise<{ status: 'valid' } | { status: 'invalid'; reason: string }> { + const ignoreService = accessor.get(IIgnoreService); + if (await ignoreService.isCopilotIgnored(URI.parse(document.uri))) { + return { + status: 'invalid', + reason: 'Document is blocked by repository policy', + }; + } + + return { status: 'valid' }; +} diff --git a/completions-sample-code/vscode-node/lib/src/util/event.ts b/completions-sample-code/vscode-node/lib/src/util/event.ts new file mode 100644 index 0000000..ccb08a0 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/event.ts @@ -0,0 +1,45 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from '../../../types/src'; +import * as lsp from 'vscode-languageserver-protocol'; +/** + * Altered to have a void return type, so eslint can flag misused promises. + */ +export interface Event<T> { + /** + * + * @param listener The listener function will be called when the event happens. + * @param thisArgs The 'this' which will be used when calling the event listener. + * @param disposables An array to which a {{Disposable}} will be added. + * @returns A disposable which unsubscribes the event listener. + */ + (listener: (e: T) => void, thisArgs?: unknown, disposables?: Disposable[]): Disposable; +} + +/** + * Altered to use the above Event interface. + */ +export class Emitter<T> extends lsp.Emitter<T> { + override get event(): Event<T> { + return super.event; + } +} + +/** + * Transforms an event by applying a transformation function to the event's value. + * Mostly useful for tranforming native VS Code events into our own. + * If the transformation function returns `undefined`, the listener will not be called. + */ +export function transformEvent<T, R extends object>(event: Event<T>, transform: (value: T) => R | undefined): Event<R> { + return (listener, thisArgs, disposables) => { + if (thisArgs) { listener = listener.bind(thisArgs); } + const wrappedListener = (value: T) => { + const transformed = transform(value); + if (transformed !== undefined) { listener(transformed); } + }; + return event(wrappedListener, undefined, disposables); + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/util/map.ts b/completions-sample-code/vscode-node/lib/src/util/map.ts new file mode 100644 index 0000000..cef77dc --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/map.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export function setDefault<K, V>(map: Map<K, V>, key: K, defaultValue: (key: K) => V) { + let value: V | undefined = map.get(key); + if (value === undefined) { + value = defaultValue(key); + map.set(key, value); + } + return value; +} diff --git a/completions-sample-code/vscode-node/lib/src/util/priorityQueue.ts b/completions-sample-code/vscode-node/lib/src/util/priorityQueue.ts new file mode 100644 index 0000000..f375370 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/priorityQueue.ts @@ -0,0 +1,133 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +type PrioritizedItem<T> = { + item: T; + priority: number; +}; + +/** + * A priority queue implementation using a binary heap. + */ +export class PriorityQueue<T> { + private heap: PrioritizedItem<T>[]; + + constructor(items?: PrioritizedItem<T>[]) { + this.heap = items ? [...items] : []; + if (this.heap.length > 0) { + // Build the heap from the initial items + for (let i = Math.floor(this.heap.length / 2) - 1; i >= 0; i--) { + this.siftDown(i); + } + } + } + + get size(): number { + return this.heap.length; + } + + /** + * Inserts an item into the queue with the given priority. + */ + insert(item: T, priority: number): void { + const newItem: PrioritizedItem<T> = { item, priority }; + this.heap.push(newItem); + const index = this.heap.length - 1; + this.siftUp(index); + } + + /** + * Returns the highest priority item without removing it. + * Returns null if the queue is empty. + */ + peek(): PrioritizedItem<T> | null { + if (this.heap.length === 0) { + return null; + } + return this.heap[0]; + } + + /** + * Removes and returns the highest priority item. + * Returns null if the queue is empty. + */ + pop(): PrioritizedItem<T> | null { + if (this.heap.length === 0) { + return null; + } + + const topItem = this.heap[0]; + const lastItem = this.heap.pop()!; + + if (this.heap.length > 0) { + this.heap[0] = lastItem; + this.siftDown(0); + } + + return topItem; + } + + clear(): PrioritizedItem<T>[] { + const items = this.heap; + this.heap = []; + return items; + } + + /** + * Moves an item up the heap until the heap property is satisfied. + */ + private siftUp(index: number): void { + const item = this.heap[index]; + + while (index > 0) { + const parentIndex = Math.floor((index - 1) / 2); + if (this.heap[parentIndex].priority >= item.priority) { + break; + } + + // Swap with parent + this.heap[index] = this.heap[parentIndex]; + + index = parentIndex; + } + + this.heap[index] = item; + } + + /** + * Moves an item down the heap until the heap property is satisfied. + */ + private siftDown(index: number): void { + while (index < this.size - 1) { + let maxChildIndex = index; + const leftChildIndex = 2 * index + 1; + const rightChildIndex = leftChildIndex + 1; + + // Find the child with higher priority + if (leftChildIndex < this.size && this.heap[leftChildIndex].priority > this.heap[maxChildIndex].priority) { + maxChildIndex = leftChildIndex; + } + + if ( + rightChildIndex < this.size && + this.heap[rightChildIndex].priority > this.heap[maxChildIndex].priority + ) { + maxChildIndex = rightChildIndex; + } + + if (maxChildIndex === index) { + // Heap property is satisfied + break; + } + + // Swap with the higher priority child + const item = this.heap[index]; + this.heap[index] = this.heap[maxChildIndex]; + this.heap[maxChildIndex] = item; + + index = maxChildIndex; + } + } +} diff --git a/completions-sample-code/vscode-node/lib/src/util/promiseQueue.ts b/completions-sample-code/vscode-node/lib/src/util/promiseQueue.ts new file mode 100644 index 0000000..6ee7e61 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/promiseQueue.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../../util/common/services'; + +export const ICompletionsPromiseQueueService = createServiceIdentifier<ICompletionsPromiseQueueService>('completionsPromiseQueueService'); +export interface ICompletionsPromiseQueueService { + readonly _serviceBrand: undefined; + + register(promise: Promise<unknown>): void; + flush(): Promise<void>; +} + +export class PromiseQueue implements ICompletionsPromiseQueueService { + declare _serviceBrand: undefined; + + protected promises = new Set<Promise<unknown>>(); + register(promise: Promise<unknown>) { + this.promises.add(promise); + void promise.finally(() => this.promises.delete(promise)); + } + + async flush() { + await Promise.allSettled(this.promises); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/util/runtimeMode.ts b/completions-sample-code/vscode-node/lib/src/util/runtimeMode.ts new file mode 100644 index 0000000..99271fe --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/runtimeMode.ts @@ -0,0 +1,81 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../../../../util/common/services'; + +type RuntimeFlag = 'debug' | 'verboseLogging' | 'testMode' | 'simulation'; + +export const ICompletionsRuntimeModeService = createServiceIdentifier<ICompletionsRuntimeModeService>('completionsRuntimeModeService'); +export interface ICompletionsRuntimeModeService { + readonly _serviceBrand: undefined; + + readonly flags: Record<RuntimeFlag, boolean>; + isRunningInTest(): boolean; + shouldFailForDebugPurposes(): boolean; + isDebugEnabled(): boolean; + isVerboseLoggingEnabled(): boolean; + isRunningInSimulation(): boolean; +} + +export class RuntimeMode implements ICompletionsRuntimeModeService { + declare _serviceBrand: undefined; + constructor(readonly flags: Record<RuntimeFlag, boolean>) { } + + static fromEnvironment(isRunningInTest: boolean, argv = process.argv, env = process.env): RuntimeMode { + return new RuntimeMode({ + debug: determineDebugFlag(argv, env), + verboseLogging: determineVerboseLoggingEnabled(argv, env), + testMode: isRunningInTest, + simulation: determineSimulationFlag(env), + }); + } + + isRunningInTest(): boolean { + return this.flags.testMode; + } + + shouldFailForDebugPurposes(): boolean { + return this.isRunningInTest(); + } + + isDebugEnabled(): boolean { + return this.flags.debug; + } + + isVerboseLoggingEnabled(): boolean { + return this.flags.verboseLogging; + } + + isRunningInSimulation(): boolean { + return this.flags.simulation; + } +} + +function determineDebugFlag(argv: string[], env: NodeJS.ProcessEnv): boolean { + return argv.includes('--debug') || determineEnvFlagEnabled(env, 'DEBUG'); +} + +function determineSimulationFlag(env: NodeJS.ProcessEnv): boolean { + return determineEnvFlagEnabled(env, 'SIMULATION'); +} + +function determineVerboseLoggingEnabled(argv: string[], env: NodeJS.ProcessEnv): boolean { + return ( + env['COPILOT_AGENT_VERBOSE'] === '1' || + env['COPILOT_AGENT_VERBOSE']?.toLowerCase() === 'true' || + determineEnvFlagEnabled(env, 'VERBOSE') || + determineDebugFlag(argv, env) + ); +} + +function determineEnvFlagEnabled(env: NodeJS.ProcessEnv, name: string): boolean { + for (const prefix of ['GH_COPILOT_', 'GITHUB_COPILOT_']) { + const val = env[`${prefix}${name}`]; + if (val) { + return val === '1' || val?.toLowerCase() === 'true'; + } + } + return false; +} diff --git a/completions-sample-code/vscode-node/lib/src/util/shortCircuit.ts b/completions-sample-code/vscode-node/lib/src/util/shortCircuit.ts new file mode 100644 index 0000000..d86c862 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/shortCircuit.ts @@ -0,0 +1,22 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +type ShortCircuitableFunction<A extends unknown[], R, T> = (this: T, ...args: A) => Promise<R>; + +// TODO: need to log whenever we hit this short circuit +export function shortCircuit<A extends unknown[], R, T>( + fn: ShortCircuitableFunction<A, R, T>, + shortCircuitMs: number, + shortCircuitReturn: R +): ShortCircuitableFunction<A, R, T> { + return async function (this: T, ...args: A) { + return await Promise.race([ + fn.apply(this, args), + new Promise<R>(resolve => { + setTimeout(resolve, shortCircuitMs, shortCircuitReturn); + }), + ]); + }; +} diff --git a/completions-sample-code/vscode-node/lib/src/util/subject.ts b/completions-sample-code/vscode-node/lib/src/util/subject.ts new file mode 100644 index 0000000..6152364 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/subject.ts @@ -0,0 +1,59 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Observer interface for the Subject. + */ +export interface Observer<T> { + next: (value: T) => void; + complete?: () => void; + error?: (err: unknown) => void; +} + +/** A simple implementation of an observable Subject. */ +export class Subject<T> { + private observers = new Set<Observer<T>>(); + + constructor() { } + + subscribe(observer: Observer<T>): () => void { + this.observers.add(observer); + return () => this.observers.delete(observer); + } + + next(value: T): void { + for (const observer of this.observers) { + observer.next(value); + } + } + + error(err: unknown): void { + for (const observer of this.observers) { + observer.error?.(err); + } + } + + complete(): void { + for (const observer of this.observers) { + observer.complete?.(); + } + } +} + +/** A variant of Subject that replays the last value to new subscribers. */ +export class ReplaySubject<T> extends Subject<T> { + private _value: T | undefined; + + override subscribe(observer: Observer<T>): () => void { + const subscription = super.subscribe(observer); + if (this._value !== undefined) { observer.next(this._value); } + return subscription; + } + + override next(value: T): void { + this._value = value; + super.next(value); + } +} diff --git a/completions-sample-code/vscode-node/lib/src/util/test/async.test.ts b/completions-sample-code/vscode-node/lib/src/util/test/async.test.ts new file mode 100644 index 0000000..c071444 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/test/async.test.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { Deferred } from '../async'; + +suite('Deferred', function () { + test('should resolve with the provided value', async function () { + const deferred = new Deferred<number>(); + const value = 42; + + deferred.resolve(value); + + assert.strictEqual(await deferred.promise, value); + }); + + test('should reject with the provided reason', async function () { + const deferred = new Deferred<string>(); + const reason = 'Error occurred'; + + deferred.reject(new Error(reason)); + + await assert.rejects(deferred.promise, { message: reason }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/util/test/priorityQueue.test.ts b/completions-sample-code/vscode-node/lib/src/util/test/priorityQueue.test.ts new file mode 100644 index 0000000..dd2ed36 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/test/priorityQueue.test.ts @@ -0,0 +1,134 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { PriorityQueue } from '../priorityQueue'; + +suite('PriorityQueue', function () { + test('should initialize with size 0', function () { + const queue = new PriorityQueue<string>(); + assert.equal(queue.size, 0); + }); + + test('peek should return null for empty queue', function () { + const queue = new PriorityQueue<string>(); + assert.equal(queue.peek(), null); + }); + + test('pop should return null for empty queue', function () { + const queue = new PriorityQueue<string>(); + assert.equal(queue.pop(), null); + }); + + test('should insert and peek highest priority item', function () { + const queue = new PriorityQueue<string>(); + queue.insert('low', 1); + queue.insert('high', 10); + queue.insert('medium', 5); + + const result = queue.peek(); + assert.equal(result?.item, 'high'); + assert.equal(result?.priority, 10); + assert.equal(queue.size, 3); + }); + + test('should pop items in priority order', function () { + const queue = new PriorityQueue<string>(); + queue.insert('low', 1); + queue.insert('high', 10); + queue.insert('medium', 5); + + let result = queue.pop(); + assert.equal(result?.item, 'high'); + assert.equal(result?.priority, 10); + assert.equal(queue.size, 2); + + result = queue.pop(); + assert.equal(result?.item, 'medium'); + assert.equal(result?.priority, 5); + assert.equal(queue.size, 1); + + result = queue.pop(); + assert.equal(result?.item, 'low'); + assert.equal(result?.priority, 1); + assert.equal(queue.size, 0); + + result = queue.pop(); + assert.equal(result, null); + }); + + test('should handle items with same priority', function () { + const queue = new PriorityQueue<string>(); + queue.insert('first', 5); + queue.insert('second', 5); + queue.insert('third', 1); + + // The highest priority item could be either 'first' or 'second' depending on implementation + // but we can at least ensure it's one of them with priority 5 + const result = queue.peek(); + assert.equal(result?.priority, 5); + assert.ok(result?.item === 'first' || result?.item === 'second'); + }); + + test('should handle multiple operations in sequence', function () { + const queue = new PriorityQueue<string>(); + + queue.insert('a', 1); + queue.insert('b', 2); + queue.insert('c', 3); + + assert.equal(queue.size, 3); + assert.equal(queue.peek()?.item, 'c'); + + queue.pop(); // removes 'c' + assert.equal(queue.size, 2); + assert.equal(queue.peek()?.item, 'b'); + + queue.insert('d', 10); + assert.equal(queue.peek()?.item, 'd'); + + queue.pop(); // removes 'd' + assert.equal(queue.peek()?.item, 'b'); + queue.insert('e', 1); + assert.equal(queue.peek()?.item, 'b'); + + assert.equal(queue.size, 3); + queue.pop(); + queue.pop(); + queue.pop(); + assert.equal(queue.size, 0); + assert.equal(queue.pop(), null); + }); + + test('should handle object items with custom identities', function () { + interface TestObject { + id: string; + value: number; + } + + const obj1 = { id: '1', value: 100 }; + const obj2 = { id: '2', value: 200 }; + + const queue = new PriorityQueue<TestObject>(); + queue.insert(obj1, 5); + queue.insert(obj2, 10); + + assert.equal(queue.peek()?.item, obj2); + }); + + test('should work for a large number of items', function () { + const queue = new PriorityQueue<number>(); + const n = 1000; + for (let i = 0; i < n; i++) { + queue.insert(i, i); + } + + for (let i = n - 1; i >= 0; i--) { + const result = queue.pop(); + assert.equal(result?.item, i); + assert.equal(result?.priority, i); + } + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/util/test/shortCircuit.test.ts b/completions-sample-code/vscode-node/lib/src/util/test/shortCircuit.test.ts new file mode 100644 index 0000000..58e8101 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/test/shortCircuit.test.ts @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import { shortCircuit } from '../shortCircuit'; + +suite('Test shortCircuit', function () { + const shortCircuitMs = 20; + const shortCircuitReturn = 'Short circuited'; + let clock: sinon.SinonFakeTimers; + setup(function () { + clock = sinon.useFakeTimers(); + }); + + teardown(function () { + clock.restore(); + }); + + test('returns the result of the function if it completes before the timeout', async function () { + const fn = (n: number) => Promise.resolve(`Result: ${n}`); + const shortCircuitedFn = shortCircuit(fn, shortCircuitMs, shortCircuitReturn); + const result = await shortCircuitedFn(42); + assert.strictEqual(result, 'Result: 42'); + }); + + test('returns the short circuit value if the function does not complete before the timeout', async function () { + let touched = false; + const timeout = new Promise(resolve => setTimeout(resolve, shortCircuitMs * 2)); + async function fn(n: number): Promise<string> { + await timeout; + touched = true; + return `Result: ${n}`; + } + const shortCircuitedFn = shortCircuit(fn, shortCircuitMs, shortCircuitReturn); + const promisedResult = shortCircuitedFn(42); // start the function, but don't await it because time is stopped + await clock.tickAsync(shortCircuitMs); // advance the clock by the short circuit time + const result = await promisedResult; + assert.strictEqual(result, 'Short circuited'); + assert.ok(!touched, 'at this point the function should still be processing and touched is not yet true'); + await clock.tickAsync(shortCircuitMs); // advance the clock to the function duration + assert.ok(touched, 'at this point the function should have completed and touched should be true'); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/util/test/subject.test.ts b/completions-sample-code/vscode-node/lib/src/util/test/subject.test.ts new file mode 100644 index 0000000..9b6414a --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/test/subject.test.ts @@ -0,0 +1,107 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { Observer, ReplaySubject, Subject } from '../subject'; + +suite('Subject', function () { + let subject: Subject<number>; + let observer: Observer<number>; + let nextCount: number; + let lastValue: number | undefined; + let errorCount: number; + let lastError: unknown; + let completeCount: number; + + setup(function () { + subject = new Subject<number>(); + nextCount = 0; + errorCount = 0; + completeCount = 0; + lastValue = undefined; + lastError = undefined; + observer = { + next: (value: number) => { + nextCount++; + lastValue = value; + }, + error: (err: unknown) => { + errorCount++; + lastError = err; + }, + complete: () => { + completeCount++; + }, + }; + }); + + test('should notify subscribed observers on next', function () { + subject.subscribe(observer); + subject.next(1); + assert.strictEqual(nextCount, 1); + assert.strictEqual(lastValue, 1); + }); + + test('should notify subscribed observers on error', function () { + const error = new Error('test error'); + subject.subscribe(observer); + subject.error(error); + assert.strictEqual(errorCount, 1); + assert.strictEqual(lastError, error); + }); + + test('should notify subscribed observers on complete', function () { + subject.subscribe(observer); + subject.complete(); + assert.strictEqual(completeCount, 1); + }); + + test('should not notify unsubscribed observers', function () { + const unsubscribe = subject.subscribe(observer); + unsubscribe(); + subject.next(1); + subject.error(new Error()); + subject.complete(); + + assert.strictEqual(nextCount, 0); + assert.strictEqual(errorCount, 0); + assert.strictEqual(completeCount, 0); + }); + + test('should notify multiple observers', function () { + let nextCount2 = 0; + let lastValue2: number | undefined; + const observer2 = { + next: (value: number) => { + nextCount2++; + lastValue2 = value; + }, + error: () => { }, + complete: () => { }, + }; + + subject.subscribe(observer); + subject.subscribe(observer2); + subject.next(1); + + assert.strictEqual(nextCount, 1); + assert.strictEqual(lastValue, 1); + assert.strictEqual(nextCount2, 1); + assert.strictEqual(lastValue2, 1); + }); + + suite('ReplaySubject', function () { + setup(function () { + subject = new ReplaySubject<number>(); + }); + + test('should notify late subscribed observers', function () { + subject.next(1); + subject.subscribe(observer); + assert.strictEqual(nextCount, 1); + assert.strictEqual(lastValue, 1); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/util/test/uri.test.ts b/completions-sample-code/vscode-node/lib/src/util/test/uri.test.ts new file mode 100644 index 0000000..ea429e8 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/test/uri.test.ts @@ -0,0 +1,147 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { platform } from 'os'; +import * as path from 'path'; +import { basename, dirname, fsPath, getFsPath, makeFsUri, normalizeUri } from '../uri'; + +suite('normalizeUri tests', function () { + test('returns the canonical form of a URI as a string', function () { + const result = normalizeUri('file:///C:/path/to/file'); + + assert.strictEqual(result, 'file:///c%3A/path/to/file'); + }); + + test('does not alter canonical URI strings', function () { + const result = normalizeUri('file:///c%3A/path/to/file'); + + assert.strictEqual(result, 'file:///c%3A/path/to/file'); + }); + + test('returns the original string for unparsable URIs', function () { + const result = normalizeUri('not a:// uri'); + + assert.strictEqual(result, 'not a:// uri'); + }); + + test('returns the original string for unparsable URIs in strict mode', function () { + const result = normalizeUri('c:\\path'); + + assert.strictEqual(result, 'c:\\path'); + }); +}); + +suite('URI file system tests', function () { + test('getFsPath returns the file path for file system URIs', function () { + // Drive letter will get normalized to lowercase by makeFsUri + assert.strictEqual(getFsPath(makeFsUri(__filename))?.toLowerCase(), __filename.toLowerCase()); + }); + + test('getFsPath uses the platform-specific file separator', function () { + assert.strictEqual(getFsPath('file:///some/path'), path.join(path.sep, 'some', 'path')); + }); + + test('getFsPath recognizes platform-specific absolute paths', function () { + if (platform() === 'win32') { + assert.strictEqual(getFsPath('file:///C:/Some/Path'), 'C:\\Some\\Path'); + } else { + assert.strictEqual(getFsPath('file:///C:/Some/Path'), '/C:/Some/Path'); + } + }); + + test('getFsPath supports UNC paths on Windows', function () { + if (platform() === 'win32') { + assert.strictEqual(getFsPath('file://Server/Share/Some/Path'), '\\\\Server\\Share\\Some\\Path'); + } else { + // on other platforms, this is the equivalent to smb://Server/Share/Some/Path, + // which is not a file system path + assert.strictEqual(getFsPath('file://Server/Share/Some/Path'), undefined); + } + }); + + test('getFsPath supports device paths on Windows', function () { + if (platform() !== 'win32') { this.skip(); } + + const devicePath = '\\\\.\\c:\\Some\\Path'; + + assert.strictEqual(getFsPath(makeFsUri(devicePath)), devicePath); + }); + + test('fsPath throws when the scheme does not represent a local file', function () { + assert.throws(() => fsPath('https://host.example/path'), /Copilot currently does not support URI with scheme/); + assert.throws(() => fsPath('untitled:Untitled-1'), /Copilot currently does not support URI with scheme/); + assert.ok(fsPath('vscode-notebook-cell:///path/to/file')); + assert.ok(fsPath('vscode-notebook:///path/to/file')); + assert.ok(fsPath('notebook:///path/to/file')); + }); + + test('fsPath uses the platform-specific definition of a local file', function () { + const uri = 'file://Server/Share/path'; + + if (platform() === 'win32') { + assert.strictEqual(fsPath(uri), '\\\\Server\\Share\\path'); + } else { + assert.throws(() => fsPath(uri), /Unsupported remote file path/); + } + }); +}); + +suite('dirname tests', function () { + test('dirname works for file URI', function () { + const dir = dirname('file:///path/to/file'); + assert.strictEqual(dir, 'file:///path/to'); + }); + test('dirname converts notebook URI to file dir', function () { + const notebookUri = 'vscode-notebook-cell:///path/to/file#cell-id'; + const dir = dirname(notebookUri); + assert.strictEqual(dir, 'file:///path/to'); + }); + + test('returns {uri: string} for {uri: string}', function () { + assert.deepStrictEqual(dirname({ uri: 'file:///path/to/file' }), { uri: 'file:///path/to' }); + }); +}); + +suite('basename tests', function () { + function verifyBasename(fsPath: string) { + const absolute = `file://${fsPath}`; + const pathExpected = path.basename(getFsPath(absolute) || ''); + const actual = basename(absolute); + assert.equal( + actual, + pathExpected, + `basename() returned '${actual}' but path.basename() returned '${pathExpected}'` + ); + const utilsExpected = basename(absolute); + assert.equal( + actual, + utilsExpected, + `basename() returned '${actual}' but Utils.basename() returned '${utilsExpected}'` + ); + } + + [ + '/path/to/file', + '/path/to/file?query', + '/path/to/file#anchor', + '/path/to/file?query#anchor', + '/path/with%20valid%20%25%20encoding', + '/path/with no % encoding', + '/path/with invalid %80 encoding', + '/path/to/directory/', + '/path/to/directory/?query', + '/path/to/directory/#anchor', + '/path/to/directory/?query#anchor', + '/', + '/?query', + '/#anchor', + '/?query#anchor', + ].forEach(fsPath => { + test(fsPath, function () { + verifyBasename(fsPath); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/lib/src/util/typebox.ts b/completions-sample-code/vscode-node/lib/src/util/typebox.ts new file mode 100644 index 0000000..9256c6e --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/typebox.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { Static, TSchema } from '@sinclair/typebox'; +import { Value } from '@sinclair/typebox/value'; + +/** + * Validates that the `payload` argument matches the input schema. This function will throw an exception if this doesnt match. + * Note: This function isnt indended to handle user valdiation, as access to errors, or messages will be up to you. + * + * @example + * ```ts + * const mySchema = Type.Object({ x: T.Number() }); + * + * expect(assertType(mySchema, { x: 123 })).toEqual({ x: 123 }); + * ``` + **/ +export const assertShape = <S extends TSchema>(schema: S, payload: unknown): Static<S> => { + if (Value.Check(schema, payload)) { return payload; } + + const error = `Typebox schema validation failed:\n${[...Value.Errors(schema, payload)] + .map(i => `${i.path} ${i.message}`) + .join('\n')}`; + + throw new Error(error); +}; diff --git a/completions-sample-code/vscode-node/lib/src/util/unknown.ts b/completions-sample-code/vscode-node/lib/src/util/unknown.ts new file mode 100644 index 0000000..d160ce9 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/unknown.ts @@ -0,0 +1,17 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** Type guard to check if an unknown value is an object with a given key. */ +function hasKey<K extends PropertyKey, R = unknown>(value: unknown, key: K): value is { [key in K]: R } { + return value !== null && typeof value === 'object' && key in value; +} + +/** + * Attempts to index an unknown value as an object. + * Returns undefined if the key does not exist on the object. + */ +export function getKey<K extends PropertyKey, R = unknown>(value: unknown, key: K): R | undefined { + return hasKey<K, R>(value, key) ? value[key] : undefined; +} diff --git a/completions-sample-code/vscode-node/lib/src/util/uri.ts b/completions-sample-code/vscode-node/lib/src/util/uri.ts new file mode 100644 index 0000000..47b40e5 --- /dev/null +++ b/completions-sample-code/vscode-node/lib/src/util/uri.ts @@ -0,0 +1,192 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { platform } from 'os'; +import { normalize } from 'path'; +import { dirname as VSCODE_dirname } from '../../../../../../util/vs/base/common/resources'; +import { URI } from '../../../../../../util/vs/base/common/uri'; + +type URIContainer = { readonly uri: string }; + +// Borrowed from vscode-uri internals +function decodeURIComponentGraceful(str: string): string { + try { + return decodeURIComponent(str); + } catch { + if (str.length > 3) { + return str.substring(0, 3) + decodeURIComponentGraceful(str.substring(3)); + } else { + return str; + } + } +} +const _rEncodedAsHex = /(%[0-9A-Za-z][0-9A-Za-z])+/g; +function percentDecode(str: string): string { + if (!str.match(_rEncodedAsHex)) { + return str; + } + return str.replace(_rEncodedAsHex, match => decodeURIComponentGraceful(match)); +} + +export function makeFsUri(fsPath: string): string { + if (/^[A-Za-z][A-Za-z0-9+.-]+:/.test(fsPath)) { + throw new Error('Path must not contain a scheme'); + } else if (!fsPath) { + throw new Error('Path must not be empty'); + } + return URI.file(fsPath).toString(); +} + +function parseUri(uri: URIContainer | string): URI { + if (typeof uri !== 'string') { uri = uri.uri; } + if (/^[A-Za-z]:\\/.test(uri)) { + throw new Error(`Could not parse <${uri}>: Windows-style path`); + } + try { + // Based on the regexp vscode-uri uses for parsing + const match = uri.match(/^(?:([^:/?#]+?:)?\/\/)(\/\/.*)$/); + if (match) { + return URI.parse(match[1] + match[2], true); + } else { + return URI.parse(uri, true); + } + } catch (cause) { + throw new Error(`Could not parse <${uri}>`, { cause }); + } +} + +/** + * Throw an exception if the URI is unparsable. + */ +/** @public KEEPING FOR TESTS */ +export function validateUri<T extends URIContainer | string>(uri: T): T { + parseUri(uri); + return uri; +} + +export function normalizeUri(uri: string): string { + try { + return parseUri(uri).toString(); + } catch { + // not normalizable, return as is + return uri; + } +} + +/** + * URI schemes that map to real file system paths. + */ +const fsSchemes = new Set(['file', 'notebook', 'vscode-notebook', 'vscode-notebook-cell']); + +/** + * For a file system URI, returns the corresponding file system path. Otherwise + * throws an error. + */ +export function fsPath(arg: URIContainer | string): string { + const uri = parseUri(arg); + + if (!fsSchemes.has(uri.scheme)) { + throw new Error(`Copilot currently does not support URI with scheme: ${uri.scheme}`); + } + + if (platform() === 'win32') { + let path = uri.path; + + if (uri.authority) { + path = `//${uri.authority}${uri.path}`; // UNC path + } else if (/^\/[A-Za-z]:/.test(path)) { + // omit leading slash from paths with a drive letter + path = path.substring(1); + } + return normalize(path); + } else if (uri.authority) { + throw new Error('Unsupported remote file path'); + } else { + return uri.path; + } +} + +/** + * For a file system URI, returns the corresponding file system path. Returns + * undefined otherwise. + */ +export function getFsPath(uri: URIContainer | string): string | undefined { + try { + return fsPath(uri); + } catch { + return undefined; + } +} + +/** + * Ensure a file system URI has a file: scheme. If it's not a file system URI, return undefined. + */ +export function getFsUri(uri: URIContainer | string): string | undefined { + const fsPath = getFsPath(uri); + if (fsPath) { + return URI.file(fsPath).toString(); + } +} + +/** + * Joins together multiple path components, with a URI as the base. + */ +export function joinPath(uri: string, ...paths: string[]): string; +export function joinPath(uri: URIContainer, ...paths: string[]): URIContainer; +export function joinPath(uri: URIContainer | string, ...paths: string[]): URIContainer | string; +export function joinPath(arg: URIContainer | string, ...paths: string[]): URIContainer | string { + const uri = URI.joinPath(parseUri(arg), ...paths.map(pathToURIPath)).toString(); + return typeof arg === 'string' ? uri : { uri }; +} + +function pathToURIPath(fileSystemPath: string): string { + if (isWinPath(fileSystemPath)) { + return fileSystemPath.replaceAll('\\', '/'); + } + + return fileSystemPath; +} + +/** + * Returns true if backlash proceeds any use of forward slash in the string. E.g.: + * + * - ..\path\to\file.txt is a Win path + * - C:\path\to\file.txt is a Win path + * - /unix/style/path is not + * - ../path/to/unusal\file.txt is not + */ +function isWinPath(path: string): boolean { + return /^[^/\\]*\\/.test(path); +} + +/** + * Returns the base filename (no directory path) of a URI. + */ +export function basename(uri: URIContainer | string): string { + return percentDecode( + (typeof uri === 'string' ? uri : uri.uri) + .replace(/[#?].*$/, '') + .replace(/\/$/, '') + .replace(/^.*[/:]/, '') + ); +} + +/** + * Returns the directory name of a URI. + * If the uri scheme is a notebook, will remove the fragment and change the scheme to file. + */ +export function dirname(uri: string): string; +export function dirname(uri: URIContainer): URIContainer; +export function dirname(uri: URIContainer | string): URIContainer | string; +export function dirname(arg: URIContainer | string): URIContainer | string { + const directoryName = VSCODE_dirname(parseUri(arg)); + let uri: string; + if (fsSchemes.has(directoryName.scheme) && directoryName.scheme !== 'file') { + uri = directoryName.with({ scheme: 'file', fragment: '' }).toString(); + } else { + uri = directoryName.toString(); + } + return typeof arg === 'string' ? uri : { uri }; +} diff --git a/completions-sample-code/vscode-node/prompt/jsx-runtime/jsx-runtime.ts b/completions-sample-code/vscode-node/prompt/jsx-runtime/jsx-runtime.ts new file mode 100644 index 0000000..d0283da --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/jsx-runtime/jsx-runtime.ts @@ -0,0 +1,75 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + FunctionComponent, + PromptComponentChild, + PromptElement, + PromptElementProps, + PromptFragment, +} from '../src/components/components'; + +/** + * JSX factory function called for any JSX element. + * + * @param type Type of the element: `type` is the function that instantiate a prompt component. We store it so that we can render the component later in the virtual prompt. + * @param props Properties of the element, with children + */ +function functionComponentFunction( + type: FunctionComponent, + props: PromptElementProps, + key?: string | number +): PromptElement { + let children: PromptComponentChild[] = []; + if (Array.isArray(props.children)) { + children = props.children; + } else if (props.children) { + children = [props.children]; + } + const componentProps = { ...props, children }; + if (key) { + componentProps.key = key; + } + return { type, props: componentProps }; +} + +/** + * JSX factory function called for any JSX fragment. + * It is used as the function when the jsx element is a fragment. It gets invoked from the reconciler when it encounters a fragment. + */ +function fragmentFunction(children: PromptComponentChild[]): PromptFragment { + return { type: 'f', children }; +} +fragmentFunction.isFragmentFunction = true; + +/* JSX namespace is used by TypeScript to type JSX: + * https://www.typescriptlang.org/docs/handbook/jsx.html#the-jsx-namespace + */ +export namespace JSX { + export interface IntrinsicElements { + [s: string]: unknown; + } + + export interface IntrinsicAttributes { + key?: string | number; + weight?: number; + source?: unknown; + } + + /* any type necessary for component prop types */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + export type ElementType<P = any> = FunctionComponent<P>; + export type Element = PromptElement; + + export interface ElementAttributesProperty { + props: unknown; + } + + export interface ElementChildrenAttribute { + children: unknown; + } +} + +export { fragmentFunction as Fragment, functionComponentFunction as jsx, functionComponentFunction as jsxs }; diff --git a/completions-sample-code/vscode-node/prompt/src/components/components.ts b/completions-sample-code/vscode-node/prompt/src/components/components.ts new file mode 100644 index 0000000..071dea9 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/components/components.ts @@ -0,0 +1,163 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DataConsumer, Dispatch, StateUpdater, TypePredicate } from './hooks'; +import { TokenizerName } from '../tokenization'; +import { CancellationToken } from 'vscode-languageserver-protocol'; + +// --------- Prompt component types + +export type PromptComponentChild = PromptElement | string | number | undefined; + +type PromptComponentChildren = PromptComponentChild[] | PromptComponentChild; + +interface PromptAttributes { + [key: string]: unknown; + key?: string | number; + weight?: number; + source?: unknown; +} + +export type PromptElementProps<P = object> = P & Readonly<PromptAttributes & { children?: PromptComponentChildren }>; + +export interface ComponentContext { + /** + * Hook to manage component state that can change over time. + * @param initialState - Initial state value or function that returns initial state + * @returns A tuple containing current state and setter function + * @example + * function Counter(props: PromptElementProps, context: ComponentContext) { + * const [count, setCount] = context.useState(0); + * return <Text>Count: {count}</Text>; + * } + */ + useState<S = undefined>(): [S | undefined, Dispatch<StateUpdater<S | undefined>>]; + useState<S>(initialState: S | (() => S)): [S, Dispatch<StateUpdater<S>>]; + + /** + * Hook to subscribe to typed external data streams with type checking. + * @param typePredicate - TypeScript type predicate function for runtime type checking + * @param consumer - Callback function that receives type-checked data + * @example + * function DataViewer(props: PromptElementProps, context: ComponentContext) { + * interface MessageData { + * message: string; + * } + * + * function isMessageData(data: unknown): data is MessageData { + * return typeof data === 'object' && data !== null && + * 'message' in data && typeof (data as any).message === 'string'; + * } + * + * context.useData( + * isMessageData, + * (data) => console.log(data.message) + * ); + * } + */ + useData<T>(typePredicate: TypePredicate<T>, consumer: DataConsumer<T>): void; +} + +export interface PromptFragment { + type: 'f'; + children: PromptComponentChild[]; +} + +export interface FragmentFunction { + (children: PromptComponentChildren): PromptFragment; +} + +export interface FunctionComponent<P = PromptAttributes> { + (props: PromptElementProps<P>, context: ComponentContext): PromptComponentChildren; +} + +/** + * Data structure returned by prompt component functions and used by the `virtualize` function to construct a virtual prompt. + */ +export interface PromptElement<P = PromptAttributes> { + type: FunctionComponent<P> | FragmentFunction; + props: P & { children: PromptComponentChildren }; +} + +// --------- Prompt snapshot and rendering types +export interface PromptSnapshotNodeStatistics { + updateDataTimeMs?: number; +} + +/** + * A prompt snapshot node is a node in the virtual prompt tree in its immutable form. + */ +export interface PromptSnapshotNode { + name: string; + path: string; + value?: string; + props?: PromptElementProps; + children?: PromptSnapshotNode[]; + statistics: PromptSnapshotNodeStatistics; +} + +export interface PromptRenderer<T extends Prompt, P extends PromptRenderOptions> { + render(snapshot: PromptSnapshotNode, options: P, cancellationToken?: CancellationToken): T; +} + +export type PromptMetadata = { + renderId: number; + rendererName?: string; + tokenizer: string; + elisionTimeMs: number; + renderTimeMs: number; + updateDataTimeMs: number; + componentStatistics: ComponentStatistics[]; +}; + +export type ComponentStatistics = { + componentPath: string; + expectedTokens?: number; + actualTokens?: number; + updateDataTimeMs?: number; + // This field is only used internally, and even tho we send it to CTS it's not telemetrized + source?: unknown; +}; + +type StatusOk = { status: 'ok' }; +export type StatusNotOk = { status: 'cancelled' } | { status: 'error'; error: Error }; +export type Status = StatusOk | StatusNotOk; + +export type PromptOk = StatusOk & { + metadata: PromptMetadata; +}; +type Prompt = PromptOk | StatusNotOk; + +export interface PromptRenderOptions { + tokenizer?: TokenizerName; + delimiter?: string; +} + +// --------- Components +type TextPromptComponentChild = string | number | undefined; +interface TextPromptElementProps extends PromptElementProps { + children?: TextPromptComponentChild[] | TextPromptComponentChild; +} + +/** + * Basic component to represent text in a prompt. + */ +export function Text(props: TextPromptElementProps) { + if (props.children) { + if (Array.isArray(props.children)) { + return props.children.join(''); + } + + return props.children; + } + return; +} + +/** + * Basic component to represent a group of components that gets elided all together or not at all. + */ +export function Chunk(props: PromptElementProps) { + return props.children; +} diff --git a/completions-sample-code/vscode-node/prompt/src/components/hooks.ts b/completions-sample-code/vscode-node/prompt/src/components/hooks.ts new file mode 100644 index 0000000..a39c6aa --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/components/hooks.ts @@ -0,0 +1,67 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type Dispatch<A> = (value: A) => void; +export type StateUpdater<S> = S | ((prevState: S) => S); + +export class UseState { + private currentIndex: number = 0; + private stateChanged: boolean = false; + + constructor(private readonly states: unknown[]) { } + + useState<S = undefined>(): [S | undefined, Dispatch<StateUpdater<S | undefined>>]; + useState<S>(initialState: S | (() => S)): [S, Dispatch<StateUpdater<S>>]; + useState<S>(initialState?: S | (() => S)): [S | undefined, Dispatch<StateUpdater<S | undefined>>] { + const index = this.currentIndex; + + // Initialize state if not exists + if (this.states[index] === undefined) { + const initial = typeof initialState === 'function' ? (initialState as () => S)() : initialState; + this.states[index] = initial; + } + + const setState = (newState: StateUpdater<S | undefined>) => { + const nextState = + typeof newState === 'function' ? (newState as (prevState: S) => S)(this.states[index] as S) : newState; + this.states[index] = nextState; + this.stateChanged = true; + }; + + this.currentIndex++; + return [this.states[index] as S, setState]; + } + + hasChanged(): boolean { + return this.stateChanged; + } +} + +export type TypePredicate<T> = (data: unknown) => data is T; +export type DataConsumer<T> = (data: T) => void | Promise<void>; + +export class UseData { + private consumers: DataConsumer<unknown>[] = []; + + constructor(private readonly measureUpdateTime: (updateTimeMs: number) => void) { } + + useData<T>(typePredicate: TypePredicate<T>, consumer: DataConsumer<T>): void { + this.consumers.push((data: unknown) => { + if (typePredicate(data)) { + return consumer(data); + } + }); + } + + async updateData(data: unknown) { + if (this.consumers.length > 0) { + const start = performance.now(); + for (const consumer of this.consumers) { + await consumer(data); + } + this.measureUpdateTime(performance.now() - start); + } + } +} diff --git a/completions-sample-code/vscode-node/prompt/src/components/reconciler.ts b/completions-sample-code/vscode-node/prompt/src/components/reconciler.ts new file mode 100644 index 0000000..1af7bd8 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/components/reconciler.ts @@ -0,0 +1,270 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken } from 'vscode-languageserver-protocol'; +import { + FragmentFunction, + FunctionComponent, + type ComponentContext, + type PromptComponentChild, + type PromptElement, + type PromptElementProps, +} from './components'; +import { DataConsumer, Dispatch, StateUpdater, TypePredicate, UseData, UseState } from './hooks'; +import { DataPipe } from './virtualPrompt'; + +/** + * A virtual prompt node is an in-memory representation of a prompt component in its rendered form. + * It is constructed from a `PromptElement` and contains the name of the component that it was constructed from, and resolved external context and state. + */ +export type VirtualPromptNode = { + name: string; + path: string; + props?: PromptElementProps; + children?: VirtualPromptNode[]; + component?: PromptComponentChild; + lifecycle?: PromptElementLifecycle; +}; + +type VirtualPromptNodeChild = VirtualPromptNode | undefined; + +/** + * Translate a `PromptComponentChild` object into a virtual prompt node. + */ + +export class VirtualPromptReconciler { + private lifecycleData: Map<string, PromptElementLifecycleData> = new Map(); + private vTree: VirtualPromptNode | undefined; + + constructor(prompt: PromptElement) { + // Initial virtualization + this.vTree = this.virtualizeElement(prompt, '$', 0); + } + + reconcile(cancellationToken?: CancellationToken): VirtualPromptNode | undefined { + if (!this.vTree) { + throw new Error('No tree to reconcile, make sure to pass a valid prompt'); + } + if (cancellationToken?.isCancellationRequested) { + return this.vTree; + } + this.vTree = this.reconcileNode(this.vTree, '$', 0, cancellationToken); + return this.vTree; + } + + private reconcileNode( + node: VirtualPromptNode, + parentNodePath: string, + nodeIndex: number, + cancellationToken?: CancellationToken + ): VirtualPromptNodeChild { + // If the node has no children or does not have a lifecycle, return it as is (primitive nodes) + if (!node.children && !node.lifecycle) { return node; } + + let newNode: VirtualPromptNodeChild = node; + + const needsReconciliation = node.lifecycle?.isRemountRequired(); + + // If the node needs reconciliation, virtualize it again + if (needsReconciliation) { + const oldChildrenPaths = this.collectChildPaths(node); + newNode = this.virtualizeElement(node.component, parentNodePath, nodeIndex); + const newChildrenPaths = this.collectChildPaths(newNode); + this.cleanupState(oldChildrenPaths, newChildrenPaths); + // Otherwise, check if the children need reconciliation + } else if (node.children) { + const children: VirtualPromptNode[] = []; + for (let i = 0; i < node.children.length; i++) { + const child = node.children[i]; + if (child) { + const reconciledChild = this.reconcileNode(child, node.path, i, cancellationToken); + if (reconciledChild !== undefined) { + children.push(reconciledChild); + } + } + } + newNode.children = children; + } + + return newNode; + } + + private virtualizeElement( + component: PromptComponentChild, + parentNodePath: string, + nodeIndex: number + ): VirtualPromptNodeChild { + if (typeof component === 'undefined') { + return undefined; + } + + if (typeof component === 'string' || typeof component === 'number') { + return { + name: typeof component, + path: `${parentNodePath}[${nodeIndex}]`, + props: { value: component }, + component, + }; + } + + if (isFragmentFunction(component.type)) { + const fragment = component.type(component.props.children); + const indexIndicator = parentNodePath !== '$' ? `[${nodeIndex}]` : ``; + const componentPath = `${parentNodePath}${indexIndicator}.${fragment.type}`; + const children = fragment.children.map((c, i) => this.virtualizeElement(c, componentPath, i)); + this.ensureUniqueKeys(children); + return { + name: fragment.type, + path: componentPath, + children: children.flat().filter(c => c !== undefined), + component, + }; + } + + return this.virtualizeFunctionComponent(parentNodePath, nodeIndex, component, component.type); + } + + private virtualizeFunctionComponent( + parentNodePath: string, + nodeIndex: number, + component: PromptElement, + functionComponent: FunctionComponent + ) { + const indexIndicator = component.props.key ? `["${component.props.key}"]` : `[${nodeIndex}]`; + const componentPath = `${parentNodePath}${indexIndicator}.${functionComponent.name}`; + const lifecycle = new PromptElementLifecycle(this.getOrCreateLifecycleData(componentPath)); + const element = functionComponent(component.props, lifecycle); + + const elementToVirtualize = Array.isArray(element) ? element : [element]; + const virtualizedChildren = elementToVirtualize.map((e, i) => this.virtualizeElement(e, componentPath, i)); + const children = virtualizedChildren.flat().filter(e => e !== undefined); + this.ensureUniqueKeys(children); + return { + name: functionComponent.name, + path: componentPath, + props: component.props, + children, + component, + lifecycle, + }; + } + + private ensureUniqueKeys(nodes: VirtualPromptNodeChild[]) { + const keyCount = new Map<string | number, number>(); + for (const node of nodes) { + if (!node) { continue; } + const key = node.props?.key; + if (key) { + keyCount.set(key, (keyCount.get(key) || 0) + 1); + } + } + // Find all duplicates + const duplicates = Array.from(keyCount.entries()) + .filter(([_, count]) => count > 1) + .map(([key]) => key); + if (duplicates.length > 0) { + throw new Error(`Duplicate keys found: ${duplicates.join(', ')}`); + } + } + + private collectChildPaths(node: VirtualPromptNode | undefined) { + const paths: string[] = []; + if (node?.children) { + for (const child of node.children) { + if (child) { + paths.push(child.path); + paths.push(...this.collectChildPaths(child)); + } + } + } + return paths; + } + + private cleanupState(oldChildrenPaths: string[], newChildrenPaths: string[]) { + for (const path of oldChildrenPaths) { + if (!newChildrenPaths.includes(path)) { + this.lifecycleData.delete(path); + } + } + } + + private getOrCreateLifecycleData(path: string) { + if (!this.lifecycleData.has(path)) { + this.lifecycleData.set(path, new PromptElementLifecycleData([])); + } + return this.lifecycleData.get(path)!; + } + + createPipe(): DataPipe { + return { + pump: async (data: unknown) => { + await this.pumpData(data); + }, + }; + } + + private async pumpData<T>(data: T) { + if (!this.vTree) { + throw new Error('No tree to pump data into. Pumping data before initializing?'); + } + await this.recursivelyPumpData(data, this.vTree); + } + + private async recursivelyPumpData<T>(data: T, node: VirtualPromptNode) { + if (!node) { + throw new Error(`Can't pump data into undefined node.`); + } + await node.lifecycle?.dataHook.updateData(data); + for (const child of node.children || []) { + await this.recursivelyPumpData(data, child); + } + } +} + +class PromptElementLifecycleData { + state: unknown[]; + _updateTimeMs: number; + + constructor(state: unknown[]) { + this.state = state; + this._updateTimeMs = 0; + } + + getUpdateTimeMsAndReset() { + const value = this._updateTimeMs; + this._updateTimeMs = 0; + return value; + } +} + +class PromptElementLifecycle implements ComponentContext { + private readonly stateHook: UseState; + readonly dataHook: UseData; + + constructor(readonly lifecycleData: PromptElementLifecycleData) { + this.stateHook = new UseState(lifecycleData.state); + this.dataHook = new UseData((updateTimeMs: number) => { + lifecycleData._updateTimeMs = updateTimeMs; + }); + } + + useState<S = undefined>(): [S | undefined, Dispatch<StateUpdater<S | undefined>>]; + useState<S>(initialState: S | (() => S)): [S, Dispatch<StateUpdater<S>>]; + useState<S>(initialState?: S | (() => S)): [S | undefined, Dispatch<StateUpdater<S | undefined>>] { + return this.stateHook.useState(initialState); + } + + useData<T>(typePredicate: TypePredicate<T>, consumer: DataConsumer<T>): void { + this.dataHook.useData(typePredicate, consumer); + } + + isRemountRequired(): boolean { + return this.stateHook.hasChanged(); + } +} + +function isFragmentFunction(element: FragmentFunction | FunctionComponent): element is FragmentFunction { + return typeof element === 'function' && 'isFragmentFunction' in element; +} diff --git a/completions-sample-code/vscode-node/prompt/src/components/virtualPrompt.ts b/completions-sample-code/vscode-node/prompt/src/components/virtualPrompt.ts new file mode 100644 index 0000000..8a92f26 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/components/virtualPrompt.ts @@ -0,0 +1,90 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { PromptElement, PromptSnapshotNode, Status } from './components'; +import { VirtualPromptNode, VirtualPromptReconciler } from './reconciler'; +import { CancellationToken } from 'vscode-languageserver-protocol'; + +type PromptSnapshot = Status & { snapshot: PromptSnapshotNode | undefined }; + +/** + * The `VirtualPrompt` class holds the in-memory representation of the prompt, and is responsible for updating it with context, and generating immutable snapshots which can be passed to a prompt renderer. + */ +export class VirtualPrompt { + private reconciler: VirtualPromptReconciler; + + constructor(prompt: PromptElement) { + this.reconciler = new VirtualPromptReconciler(prompt); + } + + private snapshotNode( + node: VirtualPromptNode, + cancellationToken?: CancellationToken + ): PromptSnapshotNode | 'cancelled' | undefined { + if (!node) { + return; + } + + if (cancellationToken?.isCancellationRequested) { + return 'cancelled'; + } + + const children = []; + for (const child of node.children ?? []) { + const result = this.snapshotNode(child, cancellationToken); + if (result === 'cancelled') { + return 'cancelled'; + } + if (result !== undefined) { + children.push(result); + } + } + + return { + value: node.props?.value?.toString(), + name: node.name, + path: node.path, + props: node.props, + children, + statistics: { + updateDataTimeMs: node.lifecycle?.lifecycleData.getUpdateTimeMsAndReset(), + }, + }; + } + + snapshot(cancellationToken?: CancellationToken): PromptSnapshot { + try { + const vTree = this.reconciler.reconcile(cancellationToken); + + if (cancellationToken?.isCancellationRequested) { + return { snapshot: undefined, status: 'cancelled' }; + } + + if (!vTree) { + throw new Error('Invalid virtual prompt tree'); + } + + const snapshotNode = this.snapshotNode(vTree, cancellationToken); + + if (snapshotNode === 'cancelled' || cancellationToken?.isCancellationRequested) { + return { snapshot: undefined, status: 'cancelled' }; + } + + return { snapshot: snapshotNode, status: 'ok' }; + } catch (e) { + return { snapshot: undefined, status: 'error', error: e as Error }; + } + } + + createPipe(): DataPipe { + return this.reconciler.createPipe(); + } +} +/** + * A data pipe is a one-way channel to get external data into the prompt. Pumping unsupported data types into the pipe will result in no-op. + */ +export interface DataPipe { + pump(data: unknown): Promise<void>; +} diff --git a/completions-sample-code/vscode-node/prompt/src/components/walker.ts b/completions-sample-code/vscode-node/prompt/src/components/walker.ts new file mode 100644 index 0000000..a21bf0b --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/components/walker.ts @@ -0,0 +1,115 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Chunk, PromptSnapshotNode } from './components'; + +/** + * Represents the context during the traversal of a prompt snapshot tree. + * This context is passed to every node and can be modified by transformers. + */ +interface WalkContext { + /** + * Context properties that can be added by custom transformers. + */ + [key: string]: unknown; +} + +/** + * A function that transforms the walking context as the tree is traversed. + * Transformers are applied in sequence before visiting each node. + * + * @param node - The current node being visited + * @param parent - The parent of the current node (undefined for root) + * @param context - The current context + * @returns A new context to be used for this node and its children + */ +export type WalkContextTransformer = ( + node: PromptSnapshotNode, + parent: PromptSnapshotNode | undefined, + context: WalkContext +) => WalkContext; + +/** + * A utility class for traversing a prompt snapshot tree. + * The walker applies transformers to modify the context at each node + * and calls a visitor function with the transformed context. + */ +export class SnapshotWalker { + /** + * Creates a new SnapshotWalker. + * + * @param snapshot - The root node of the snapshot tree to walk + * @param transformers - Optional array of context transformers to apply during traversal + */ + constructor( + private readonly snapshot: PromptSnapshotNode, + private readonly transformers: WalkContextTransformer[] = defaultTransformers() + ) { } + + /** + * Walks the snapshot tree and applies the visitor function to each node. + * + * @param visitor - Function called for each node during traversal. Return false to skip traversing children. + * @param options - Optional configuration for the walk + */ + walkSnapshot( + visitor: (n: PromptSnapshotNode, parent: PromptSnapshotNode | undefined, context: WalkContext) => boolean + ) { + this.walkSnapshotNode(this.snapshot, undefined, visitor, {}); + } + + private walkSnapshotNode( + node: PromptSnapshotNode, + parent: PromptSnapshotNode | undefined, + visitor: (n: PromptSnapshotNode, parent: PromptSnapshotNode | undefined, context: WalkContext) => boolean, + context: WalkContext + ) { + // Apply all transformers to create the new context for this node + const newContext = this.transformers.reduce((ctx, transformer) => transformer(node, parent, ctx), { ...context }); + + // Visit the node with the transformed context + const accept = visitor(node, parent, newContext); + if (!accept) { + return; + } + + // Process children with the new context + for (const child of node.children ?? []) { + this.walkSnapshotNode(child, node, visitor, newContext); + } + } +} + +export function defaultTransformers(): WalkContextTransformer[] { + return [ + // Weight transformer - computes the weight of the current relative to the parent + (node, _, context) => { + if (context.weight === undefined) { + context.weight = 1; + } + const weight = node.props?.weight ?? 1; + const clampedWeight = typeof weight === 'number' ? Math.max(0, Math.min(1, weight)) : 1; + return { ...context, weight: clampedWeight * (context.weight as number) }; + }, + // Chunk transformer + (node, _, context) => { + if (node.name === Chunk.name) { + // Initialize chunk set if it doesn't exist + const chunks = context.chunks ? new Set<string>(context.chunks as Set<string>) : new Set<string>(); + // Add current node path to the set + chunks.add(node.path); + return { ...context, chunks }; + } + return context; + }, + // Source transformer + (node, _, context) => { + if (node.props?.source !== undefined) { + return { ...context, source: node.props.source }; + } + return context; + }, + ]; +} diff --git a/completions-sample-code/vscode-node/prompt/src/error.ts b/completions-sample-code/vscode-node/prompt/src/error.ts new file mode 100644 index 0000000..ee7c8a5 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/error.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +export class CopilotPromptLoadFailure extends Error { + readonly code = 'CopilotPromptLoadFailure'; + constructor(message: string, cause?: unknown) { + super(message, { cause }); + } +} diff --git a/completions-sample-code/vscode-node/prompt/src/fileLoader.ts b/completions-sample-code/vscode-node/prompt/src/fileLoader.ts new file mode 100644 index 0000000..b34b088 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/fileLoader.ts @@ -0,0 +1,31 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as fs from 'node:fs/promises'; +import path from 'node:path'; + +export async function readFile(filename: string): Promise<Uint8Array> { + return await fs.readFile(locateFile(filename)); +} + +export function locateFile(filename: string): string { + // construct a path that works both for the TypeScript source, which lives under `/src`, and for + // the transpiled JavaScript, which lives under `/dist` + return path.resolve( + path.extname(__filename) === '.ts' ? path.join(locationInPath(path.dirname(__dirname), 'src'), '..', 'dist') : locationInPath(__dirname, 'dist'), + filename + ); +} + +function locationInPath(filePath: string, directoryName: string): string { + let p = filePath; + while (path.basename(p) !== directoryName) { + if (path.dirname(p) === p) { + return filePath; + } + p = path.dirname(p); + } + return p; +} + diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/classes.ts b/completions-sample-code/vscode-node/prompt/src/indentation/classes.ts new file mode 100644 index 0000000..417fe4a --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/classes.ts @@ -0,0 +1,132 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type IndentationTree<L> = TopNode<L> | VirtualNode<L> | LineNode<L> | BlankNode<L>; +export type IndentationSubTree<L> = Exclude<IndentationTree<L>, TopNode<L>>; + +interface NodeBase<L> { + label?: L; + subs: IndentationSubTree<L>[]; +} + +/** + * Virtual nodes represent groupings are not directly visible in indentation. + **/ +export interface VirtualNode<L> extends NodeBase<L> { + type: 'virtual'; + indentation: number; +} + +export interface TopNode<L> extends NodeBase<L> { + type: 'top'; + indentation: -1; +} + +/** + * A line of source code and its sub-nodes + * */ +export interface LineNode<L> extends NodeBase<L> { + type: 'line'; + indentation: number; + lineNumber: number; + sourceLine: string; +} + +/** + * A blank line + */ +interface BlankNode<L> extends NodeBase<L> { + type: 'blank'; + lineNumber: number; + subs: never[]; // Type trick to make it easier to code +} + +/** Construct a virtual node */ +export function virtualNode<L>(indentation: number, subs: IndentationSubTree<L>[], label?: L): VirtualNode<L> { + return { type: 'virtual', indentation, subs, label }; +} + +/** Construct a line node */ +export function lineNode<L>( + indentation: number, + lineNumber: number, + sourceLine: string, + subs: IndentationSubTree<L>[], + label?: L +): LineNode<L> { + if (sourceLine === '') { + throw new Error('Cannot create a line node with an empty source line'); + } + return { type: 'line', indentation, lineNumber, sourceLine, subs, label }; +} + +/** Return a blank node */ +export function blankNode(line: number): BlankNode<never> { + return { type: 'blank', lineNumber: line, subs: [] }; +} + +/** Return a node representing the top node */ +export function topNode<L>(subs?: IndentationSubTree<L>[]): TopNode<L> { + return { + type: 'top', + indentation: -1, + subs: subs ?? [], + }; +} + +export function isBlank<L>(tree: IndentationTree<L>): tree is BlankNode<L> { + return tree.type === 'blank'; +} + +export function isLine<L>(tree: IndentationTree<L>): tree is LineNode<L> { + return tree.type === 'line'; +} + +export function isVirtual<L>(tree: IndentationTree<L>): tree is VirtualNode<L> { + return tree.type === 'virtual'; +} + +export function isTop<L>(tree: IndentationTree<L>): tree is TopNode<L> { + return tree.type === 'top'; +} + +/** + * Return the tree which consists of everything up to the line node with the + * given number. All later siblings of that line node, recursively, are removed. + * + * This function does not assume the line numbers appear contiguously, but will + * return anything before the numbered line, whether its line number is greater + * or not. + * + * This is destructive and modifies the tree. + */ +export function cutTreeAfterLine(tree: IndentationTree<unknown>, lineNumber: number) { + function cut(tree: IndentationTree<unknown>): boolean { + if (!isVirtual(tree) && !isTop(tree) && tree.lineNumber === lineNumber) { + tree.subs = []; + return true; + } + for (let i = 0; i < tree.subs.length; i++) { + if (cut(tree.subs[i])) { + tree.subs = tree.subs.slice(0, i + 1); + return true; + } + } + return false; + } + cut(tree); +} + +/** + * A type expressing that JSON.parse(JSON.stringify(x)) === x. + */ +export type JsonStable = string | number | JsonStable[] | { [key: string]: JsonStable }; + +/** + * Return a deep duplicate of the tree -- this will only work if the labels can be stringified to parseable JSON. + */ +export function duplicateTree<L extends JsonStable>(tree: IndentationTree<L>): IndentationTree<L> { + return <IndentationTree<L>>JSON.parse(JSON.stringify(tree)); +} diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/description.ts b/completions-sample-code/vscode-node/prompt/src/indentation/description.ts new file mode 100644 index 0000000..ee6ff4d --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/description.ts @@ -0,0 +1,162 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IndentationTree, isBlank, isLine, isTop, isVirtual, JsonStable, LineNode } from './classes'; +import { foldTree } from './manipulation'; + +/** + * Format only the given line node, and *NOT* its subnodes. + * This essentially comprise indentation and a trailing newline. + */ +export function deparseLine<T>(node: LineNode<T>): string { + return ' '.repeat(node.indentation) + node.sourceLine + '\n'; +} + +/** + * Return a flat string representation of the indentation tree. + */ +export function deparseTree<L>(tree: IndentationTree<L>): string { + function accumulator(tree: IndentationTree<L>, accum: string): string { + let str = ''; + if (isLine(tree)) { + str = deparseLine(tree); + } else if (isBlank(tree)) { + str = '\n'; + } + return accum + str; + } + return foldTree(tree, '', accumulator, 'topDown'); +} + +/** + * Return a list of flat strings whose concatenation equals `deparseTree`. + * The source is cut at the lines whose labels appear in `cutAt`. In other + * words, if a node has a labelled `A` that appears in `cutAt`, then there will + * be at least three strings in the result: the concatenation of lines before + * the node `A`, the lines covered by node `A`, and lines after the node `A`. + * + * FIXME: The cuts are *not* applied recursively: If e.g. node `A` has a + * sub-node labelled `B` which is also in `cutAt`, then the result will still + * contain only a single string for node `A`. + * + */ +export function deparseAndCutTree<L>(tree: IndentationTree<L>, cutAt: L[]): { label: L | undefined; source: string }[] { + const cutAtSet = new Set(cutAt); + const cuts: { label: L | undefined; source: string }[] = []; + let curUndef = ''; + // Reimplement visitTree to avoid descending into cut nodes. + function visit(tree: IndentationTree<L>) { + if (tree.label !== undefined && cutAtSet.has(tree.label)) { + if (curUndef !== '') { + cuts.push({ label: undefined, source: curUndef }); + } + cuts.push({ + label: tree.label, + source: deparseTree(tree), + }); + curUndef = ''; + } else { + if (isLine(tree)) { + curUndef += deparseLine(tree); + } + tree.subs.forEach(visit); + } + } + visit(tree); + if (curUndef !== '') { + cuts.push({ label: undefined, source: curUndef }); + } + return cuts; +} + +/** + * Return a readable string representation of the tree. + * + * The output is closely related to building trees using the helper functions in + * `indentation.test.ts`. + */ +export function describeTree<L>(tree: IndentationTree<L>, indent = 0): string { + const ind = ' '.repeat(indent); + if (tree === undefined) { + return 'UNDEFINED NODE'; + } + let children: string; + if (tree.subs === undefined) { + children = 'UNDEFINED SUBS'; + } else { + children = tree.subs.map(child => describeTree(child, indent + 2)).join(',\n'); + } + if (children === '') { + children = '[]'; + } else { + children = `[\n${children}\n ${ind}]`; + } + const prefix = (isVirtual(tree) || isTop(tree) ? ' ' : String(tree.lineNumber).padStart(3, ' ')) + `: ${ind}`; + const labelString = tree.label === undefined ? '' : JSON.stringify(tree.label); + if (isVirtual(tree) || isTop(tree)) { + return `${prefix}vnode(${tree.indentation}, ${labelString}, ${children})`; + } else if (isBlank(tree)) { + return `${prefix}blank(${labelString ?? ''})`; + } else { + return `${prefix}lnode(${tree.indentation}, ${labelString}, ${JSON.stringify(tree.sourceLine)}, ${children})`; + } +} + +/** + * Return a string that mimics the call that would construct the tree + * This is less readable than describeTree, but useful to write code. + */ +export function encodeTree<T extends JsonStable>(tree: IndentationTree<T>, indent = ''): string { + const labelString = tree.label === undefined ? '' : `, ${JSON.stringify(tree.label)}`; + + const subString = + !isBlank(tree) && tree.subs.length > 0 + ? `[\n${tree.subs.map(node => encodeTree(node, indent + ' ')).join(', \n')}\n${indent}]` + : '[]'; + + switch (tree.type) { + case 'blank': + return `${indent}blankNode(${tree.lineNumber}${labelString})`; + case 'top': + return `topNode(${subString}${labelString})`; + case 'virtual': + return `${indent}virtualNode(${tree.indentation}, ${subString}${labelString})`; + case 'line': + return `${indent}lineNode(${tree.indentation}, ${tree.lineNumber}, "${tree.sourceLine}", ${subString}${labelString})`; + } +} + +/** + * Return the first line number of the given tree. + */ +export function firstLineOf<L>(tree: IndentationTree<L>): number | undefined { + if (isLine(tree) || isBlank(tree)) { + return tree.lineNumber; + } + for (const sub of tree.subs) { + const firstLine = firstLineOf(sub); + if (firstLine !== undefined) { + return firstLine; + } + } + return undefined; +} + +/** + * Return the last line number of the given tree. + */ +export function lastLineOf<L>(tree: IndentationTree<L>): number | undefined { + let lastLine: number | undefined = undefined; + let i = tree.subs.length - 1; + while (i >= 0 && lastLine === undefined) { + lastLine = lastLineOf(tree.subs[i]); + i--; + } + if (lastLine === undefined && !isVirtual(tree) && !isTop(tree)) { + return tree.lineNumber; + } else { + return lastLine; + } +} diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/index.ts b/completions-sample-code/vscode-node/prompt/src/indentation/index.ts new file mode 100644 index 0000000..a7907b0 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/index.ts @@ -0,0 +1,17 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { processJava } from './java'; +import { processMarkdown } from './markdown'; +import { registerLanguageSpecificParser } from './parsing'; + +registerLanguageSpecificParser('markdown', processMarkdown); +registerLanguageSpecificParser('java', processJava); + +export * from './classes'; +export * from './description'; +export * from './manipulation'; +export * from './parsing'; + diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/java.ts b/completions-sample-code/vscode-node/prompt/src/indentation/java.ts new file mode 100644 index 0000000..37f97bc --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/java.ts @@ -0,0 +1,72 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IndentationTree, isBlank } from './classes'; +import { visitTree } from './manipulation'; +import { + LabelRule, + buildLabelRules, + combineClosersAndOpeners, + flattenVirtual, + labelLines, + labelVirtualInherited, +} from './parsing'; + +/** + * Java labels. + * + * * package: A package declaration; + * * import: An import stament + * * comment_single: Single-line comments starting with // + * * comment_multi: Multi-line comments starting with /*, or a vnode of + * multiple single-line comments. + * * annotation: A line starting with "@". Note that fields are habitually + * declared on one line, even if they have an annotation. In this case, the + * field will have the label "annotation" rather than "member". + * * closeBrace: A closing brace alone on a line. + * * member: Anything inside a class or interface that does not have a more + * specific label. + */ +const _javaLabelRules = { + package: /^package /, + import: /^import /, + class: /\bclass /, + interface: /\binterface /, + javadoc: /^\/\*\*/, + comment_multi: /^\/\*[^*]/, + comment_single: /^\/\//, + annotation: /^@/, + opener: /^[[({]/, + closer: /^[\])}]/, +} as const; +const javaLabelRules: LabelRule<string>[] = buildLabelRules(_javaLabelRules); + +/** + * processJava(parseRaw(text)) is supposed to serve as superior alternative to alternative parseTree(text, "generic") + */ +export function processJava<L>(originalTree: IndentationTree<L>): IndentationTree<L | string> { + let tree = originalTree as IndentationTree<L | string>; + labelLines(tree, javaLabelRules); + tree = combineClosersAndOpeners(tree); + tree = flattenVirtual(tree); + labelVirtualInherited(tree); + // Label all non-labelled subs of class and interface as member. + // We also relabel annotations that are direct subs of class or interface as + // member. + visitTree( + tree, + (tree: IndentationTree<L | string>) => { + if (tree.label === 'class' || tree.label === 'interface') { + for (const sub of tree.subs) { + if (!isBlank(sub) && (sub.label === undefined || sub.label === 'annotation')) { + sub.label = 'member'; + } + } + } + }, + 'bottomUp' + ); + return tree; +} diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/manipulation.ts b/completions-sample-code/vscode-node/prompt/src/indentation/manipulation.ts new file mode 100644 index 0000000..aa05714 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/manipulation.ts @@ -0,0 +1,190 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IndentationSubTree, IndentationTree, TopNode, isTop, isVirtual, topNode } from './classes'; + +/** + * Clear all labels (and their types) from the tree. + * This will modify the tree in place, or return a retyped tree. + */ +export function clearLabels<L>(tree: IndentationTree<L>): IndentationTree<never> { + visitTree( + tree, + (tree: IndentationTree<L>) => { + tree.label = undefined; + }, + 'bottomUp' + ); + return tree as IndentationTree<never>; +} + +/** clear labels if condition is true */ +export function clearLabelsIf<L, S>( + tree: IndentationTree<L | S>, + condition: (arg: L | S) => arg is S +): IndentationTree<L> { + visitTree( + tree, + (tree: IndentationTree<L | S>) => { + tree.label = tree.label ? (condition(tree.label) ? undefined : tree.label) : undefined; + }, + 'bottomUp' + ); + return tree as IndentationTree<L>; +} + +export function mapLabels<L1, L2>( + tree: IndentationSubTree<L1>, + map: (arg: L1) => L2 | undefined +): IndentationSubTree<L2>; +export function mapLabels<L1, L2>(tree: TopNode<L1>, map: (arg: L1) => L2 | undefined): TopNode<L2>; +export function mapLabels<L1, L2>(tree: IndentationTree<L1>, map: (arg: L1) => L2 | undefined): IndentationTree<L2>; +/** + * Apply a type changing function to all labels. + * This will return a new, retyped tree. + * (For applying a type keeping function to a tree + * that modifies it in place, use `visitTree`.) + */ +export function mapLabels<L1, L2>(tree: IndentationTree<L1>, map: (arg: L1) => L2 | undefined): IndentationTree<L2> { + switch (tree.type) { + case 'line': + case 'virtual': { + const newSubs = tree.subs.map(sub => mapLabels(sub, map)); + return { ...tree, subs: newSubs, label: tree.label ? map(tree.label) : undefined }; + } + case 'blank': + return { ...tree, label: tree.label ? map(tree.label) : undefined }; + case 'top': + return { + ...tree, + subs: tree.subs.map(sub => mapLabels(sub, map)), + label: tree.label ? map(tree.label) : undefined, + }; + } +} + +/** + * Renumber the line numbers of the tree contiguously from 0 and up. + */ +export function resetLineNumbers<L>(tree: IndentationTree<L>): void { + let lineNumber = 0; + function visitor(tree: IndentationTree<L>) { + if (!isVirtual(tree) && !isTop(tree)) { + tree.lineNumber = lineNumber; + lineNumber++; + } + } + visitTree(tree, visitor, 'topDown'); +} + +/** + * Visit the tree with a function that is called on each node. + * + * If direction is topDown, then parents are visited before their children. + * If direction is bottomUp, children are visited in order before their parents, + * so that leaf nodes are visited first. + */ +export function visitTree<L>( + tree: IndentationTree<L>, + visitor: (tree: IndentationTree<L>) => void, + direction: 'topDown' | 'bottomUp' +): void { + function _visit(tree: IndentationTree<L>) { + if (direction === 'topDown') { + visitor(tree); + } + tree.subs.forEach(subtree => { + _visit(subtree); + }); + if (direction === 'bottomUp') { + visitor(tree); + } + } + _visit(tree); +} + +/** + * Visit the tree with a function that is called on each node -- + * if it returns false, children are not visited (in case of topDown), + * or the parent is not visited anymore (in case of bottomUp). + * + * If direction is topDown, then parents are visited before their children. + * If direction is bottomUp, children are visited in order before their parents, + * so that leaf nodes are visited first. + */ +export function visitTreeConditionally<L>( + tree: IndentationTree<L>, + visitor: (tree: IndentationTree<L>) => boolean, + direction: 'topDown' | 'bottomUp' +): void { + // IDEA: rewrite visitTree to reuse this code + function _visit(tree: IndentationTree<L>): boolean { + if (direction === 'topDown') { + if (!visitor(tree)) { + return false; + } + } + let shouldContinue = true; + tree.subs.forEach(subtree => { + shouldContinue = shouldContinue && _visit(subtree); + }); + if (direction === 'bottomUp') { + shouldContinue = shouldContinue && visitor(tree); + } + return shouldContinue; + } + _visit(tree); +} + +/** + * Fold an accumulator function over the tree. + * + * If direction is topDown, then parents are visited before their children. + * If direction is bottomUp, children are visited in order before their parents, + * so that leaf nodes are visited first. + */ +export function foldTree<T, L>( + tree: IndentationTree<L>, + init: T, + accumulator: (tree: IndentationTree<L>, acc: T) => T, + direction: 'topDown' | 'bottomUp' +): T { + let acc = init; + function visitor(tree: IndentationTree<L>) { + acc = accumulator(tree, acc); + } + visitTree(tree, visitor, direction); + return acc; +} + +export type Rebuilder<L> = (tree: IndentationTree<L>) => IndentationTree<L> | undefined; +/** + * Rebuild the tree from the bottom up by applying a function to each node. + * The visitor function takes a node whose children have already been rebuilt, + * and returns a new node to replace it (or undefined if it should be deleted). + * Optionally, a function can be provided to skip nodes that should just be kept + * without visiting them or their sub-nodes. + */ +export function rebuildTree<L>( + tree: IndentationTree<L>, + visitor: Rebuilder<L>, + skip?: (tree: IndentationTree<L>) => boolean +): IndentationTree<L> { + const rebuild: Rebuilder<L> = (tree: IndentationTree<L>) => { + if (skip !== undefined && skip(tree)) { + return tree; + } else { + const newSubs = tree.subs.map(rebuild).filter(sub => sub !== undefined) as IndentationSubTree<L>[]; + tree.subs = newSubs; + return visitor(tree); + } + }; + const rebuilt = rebuild(tree); + if (rebuilt !== undefined) { + return rebuilt; + } else { + return topNode(); + } +} diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/markdown.ts b/completions-sample-code/vscode-node/prompt/src/indentation/markdown.ts new file mode 100644 index 0000000..e824dea --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/markdown.ts @@ -0,0 +1,75 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IndentationTree, isBlank, LineNode, TopNode, VirtualNode } from './classes'; +import { + buildLabelRules, + flattenVirtual, + groupBlocks, + labelLines, + LabelRule, + labelVirtualInherited, +} from './parsing'; + +/** + + */ +const _MarkdownLabelRules = { + heading: /^# /, + subheading: /^## /, + subsubheading: /### /, +} as const; +const MarkdownLabelRules: LabelRule<string>[] = buildLabelRules(_MarkdownLabelRules); + +/** + * processMarkdown(parseRaw(text)) is supposed to serve as a superior alternative to parseTree(text, "generic") + */ +export function processMarkdown<L>(originalTree: IndentationTree<L>): IndentationTree<L | string> { + let tree = originalTree as IndentationTree<L | string>; + labelLines(tree, MarkdownLabelRules); + + // We'll want to refer to the tree's subs, so let the type checker know it won't be blank + if (isBlank(tree)) { + return tree; + } + + // the top level is ordered according to headings / subheadings / subsubheadings + function headingLevel(sub: IndentationTree<L | string>): number | undefined { + // 0 is the tree itself, so we start at 1 + if (sub.label === 'heading') { return 1; } + if (sub.label === 'subheading') { return 2; } + if (sub.label === 'subsubheading') { return 3; } + return undefined; + } + const currentHierarchy: (TopNode<L | string> | LineNode<L | string> | VirtualNode<L | string>)[] = [tree]; + const oldTreeSubs = [...tree.subs]; + tree.subs = []; + for (const sub of oldTreeSubs) { + const level = headingLevel(sub); + if (level === undefined || isBlank(sub)) { + currentHierarchy[currentHierarchy.length - 1].subs.push(sub); + } else { + // take care of "dangling" levels, e.g. if we have a subsubheading after a heading + while (currentHierarchy.length < level) { + currentHierarchy.push(currentHierarchy[currentHierarchy.length - 1]); + } + // add this to the parent + currentHierarchy[level - 1].subs.push(sub); + // make this the tip of the hierarchy + currentHierarchy[level] = sub; + // delete all higher levels + while (currentHierarchy.length > level + 1) { + currentHierarchy.pop(); + } + } + } + + // now group paragraphs + tree = groupBlocks(tree); + tree = flattenVirtual(tree); + labelVirtualInherited(tree); + + return tree; +} diff --git a/completions-sample-code/vscode-node/prompt/src/indentation/parsing.ts b/completions-sample-code/vscode-node/prompt/src/indentation/parsing.ts new file mode 100644 index 0000000..bbd5cd4 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/indentation/parsing.ts @@ -0,0 +1,332 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + blankNode, + IndentationSubTree, + IndentationTree, + isBlank, + isLine, + isVirtual, + lineNode, + LineNode, + TopNode, + topNode, + virtualNode, + VirtualNode, +} from './classes'; +import { clearLabelsIf, Rebuilder, rebuildTree, visitTree } from './manipulation'; + +/** + * Perform a raw indentation-tree parse of a string. This is completely + * language-agnostic and the returned tree is unlabeled. + * + * - Blank lines pertain to the top-most node that they may, as restricted + * by next non-blank line. So e.g. + * + * E + * e1 + * e2 + * + * e3 + * + * Then e1.subs = [e2], and E.subs = [ e1, blank, e3 ]. + * + */ +export function parseRaw(source: string): IndentationTree<never> { + const rawLines = source.split('\n'); + // TODO: How to handle mix of tabs and spaces? + const indentations = rawLines.map(line => line.match(/^\s*/)![0].length); + const lines = rawLines.map(line => line.trimLeft()); + function parseNode(line: number): [LineNode<never>, number] { + const [subs, nextLine] = parseSubs(line + 1, indentations[line]); + const node: LineNode<never> = lineNode(indentations[line], line, lines[line], subs); + return [node, nextLine]; + } + function parseSubs(initialLine: number, parentIndentation: number): [IndentationSubTree<never>[], number] { + let sub: IndentationTree<never> | undefined; + const subs: IndentationSubTree<never>[] = []; + let line = initialLine; + let lastBlank: number | undefined = undefined; + while (line < lines.length && (lines[line] === '' || indentations[line] > parentIndentation)) { + if (lines[line] === '') { + if (lastBlank === undefined) { + lastBlank = line; + } + line += 1; + } else { + if (lastBlank !== undefined) { + for (let i = lastBlank; i < line; i++) { + subs.push(blankNode(i)); + } + lastBlank = undefined; + } + [sub, line] = parseNode(line); + subs.push(sub); + } + } + // Trailing blanks are left for the grandparent + if (lastBlank !== undefined) { + line = lastBlank; + } + return [subs, line]; + } + const [subs, parsedLine] = parseSubs(0, -1); + let line = parsedLine; + // Special case: trailing blank lines at end of file + while (line < lines.length && lines[line] === '') { + subs.push(blankNode(line)); + line += 1; + } + if (line < lines.length) { + throw new Error(`Parsing did not go to end of file. Ended at ${line} out of ${lines.length}`); + } + return topNode(subs); +} + +type LineMatcher = (sourceLine: string) => boolean; +export interface LabelRule<L> { + matches: LineMatcher; + label: L | undefined; +} + +/** Labels the line elements of the tree in-place according to rules */ +export function labelLines<L>(tree: IndentationTree<L>, labelRules: LabelRule<L>[]): void { + function visitor(tree: IndentationTree<L>): void { + if (isLine(tree)) { + const rule = labelRules.find(rule => rule.matches(tree.sourceLine)); + if (rule) { + tree.label = rule.label; + } + } + } + visitTree(tree, visitor, 'bottomUp'); +} + +/** + * For each virtual node, if the node has only one non-blank sub, then label + * the virtual node as that sub. + */ +export function labelVirtualInherited<L>(tree: IndentationTree<L>): void { + function visitor(tree: IndentationTree<L>): void { + if (isVirtual(tree) && tree.label === undefined) { + const subs = tree.subs.filter(sub => !isBlank(sub)); + if (subs.length === 1) { + tree.label = subs[0].label; + } + } + } + visitTree(tree, visitor, 'bottomUp'); +} + +/** + * Function to convert a mapped object to a list of rules. + * This allows some type magic for extracting a label type from a mapping of rules. + */ +export function buildLabelRules<L extends { [key: string]: RegExp | LineMatcher }>(ruleMap: L): LabelRule<keyof L>[] { + return (Object.keys(ruleMap) as (keyof L)[]).map(key => { + let matches: (sourceLine: string) => boolean; + if ((ruleMap[key] as RegExp).test) { + matches = sourceLine => (ruleMap[key] as RegExp).test(sourceLine); + } else { + matches = ruleMap[key] as LineMatcher; + } + return { + matches, + label: key, + }; + }); +} + +/** + * Fills the opener and closer indentation spec + * 1. Openers alone in a line whose older sibling is a line are moved to be the first of that sibling's children, + * and their children integrated as subsequent children of their new parent. + * 2. Closers following an older sibling (maybe with blanks in between) are moved to be the last of that sibling. + * 3. If the closer in 2 has children themselves, their older siblings are wrapped in a virtual node + */ +export function combineClosersAndOpeners<L>( + tree: IndentationTree<L | 'opener' | 'closer'> +): IndentationTree<L | 'opener' | 'closer'> { + // We'll make new virtual nodes, which comprise older siblings of a closer and get a temporary label + type S = L | 'opener' | 'closer' | 'newVirtual'; + const rebuilder: Rebuilder<S> = function (tree: IndentationTree<S>) { + if ( + tree.subs.length === 0 || + tree.subs.findIndex(sub => sub.label === 'closer' || sub.label === 'opener') === -1 + ) { + return tree; + } + const newSubs: IndentationSubTree<S>[] = []; + let lastNew: TopNode<S> | VirtualNode<S> | LineNode<S> | undefined; + for (let i = 0; i < tree.subs.length; i++) { + const sub = tree.subs[i]; + const directOlderSibling = tree.subs[i - 1]; + // 1. if opener whose older sibling is a line, move to first of that sibling's children + if (sub.label === 'opener' && directOlderSibling !== undefined && isLine(directOlderSibling)) { + // Move the bracket to be the last child of it + directOlderSibling.subs.push(sub); + sub.subs.forEach(sub => directOlderSibling.subs.push(sub)); + sub.subs = []; + } + // 2. if a closer following an older sibling + else if ( + sub.label === 'closer' && + lastNew !== undefined && + (isLine(sub) || isVirtual(sub)) && + sub.indentation >= lastNew.indentation + ) { + // Move intervening blanks from newSubs to lastNew.subs + let j = newSubs.length - 1; + while (j > 0 && isBlank(newSubs[j])) { + j -= 1; + } + lastNew.subs.push(...newSubs.splice(j + 1)); + + // 3.if the closer in 2 has children themselves, their older siblings are wrapped in a virtual node to distinguish them + // Except for leading blocks of virtual nodes which have already been wrapped that way + // i.e. take the longest initial subsequence of lastNew.subs that are all labeled 'virtual' and don't wrap those again + if (sub.subs.length > 0) { + const firstNonVirtual = lastNew.subs.findIndex(sub => sub.label !== 'newVirtual'); + const subsToKeep = lastNew.subs.slice(0, firstNonVirtual); + const subsToWrap = lastNew.subs.slice(firstNonVirtual); + const wrappedSubs = + subsToWrap.length > 0 ? [virtualNode(sub.indentation, subsToWrap, 'newVirtual')] : []; + lastNew.subs = [...subsToKeep, ...wrappedSubs, sub]; + } else { + lastNew.subs.push(sub); + } + } else { + // nothing to do here, just add it normally + newSubs.push(sub); + if (!isBlank(sub)) { + lastNew = sub; + } + } + } + tree.subs = newSubs; + return tree; + }; + const returnTree = rebuildTree(tree, rebuilder); + clearLabelsIf<S, 'newVirtual'>(tree, (arg: S): arg is 'newVirtual' => arg === 'newVirtual'); + // now returnTree does not have the helper label 'newVirtual' anymore + return returnTree as IndentationTree<L | 'opener' | 'closer'>; +} + +/** + * If there are more than 1 consecutive sibling separated from others by delimiters, + * combine them into a virtual node. + * The possibly several consecutive delimiters will be put with the preceding siblings into the virtual node. + * Note that offside groupings should be done before this. + */ +export function groupBlocks<L>( + tree: IndentationTree<L>, + isDelimiter: (node: IndentationTree<L>) => boolean = isBlank, + label?: L +): IndentationTree<L> { + const rebuilder: Rebuilder<L> = function (tree: IndentationTree<L>) { + if (tree.subs.length <= 1) { + return tree; + } + const newSubs: IndentationSubTree<L>[] = []; + let nodesSinceLastFlush: IndentationSubTree<L>[] = []; + let currentBlockIndentation: number | undefined; + let lastNodeWasDelimiter = false; + + // we write to nodesSinceLastDelimiter as cache + // if we have a non-delimiter after a delimiter, we flush + // to a new virtual node appended to the newSubs array + + function flushBlockIntoNewSubs( + final: boolean = false // if final, only wrap in virtual if there are newSubs already + ): void { + if (currentBlockIndentation !== undefined && (newSubs.length > 0 || !final)) { + const virtual = virtualNode(currentBlockIndentation, nodesSinceLastFlush, label); + newSubs.push(virtual); + } else { + nodesSinceLastFlush.forEach(node => newSubs.push(node)); + } + } + + for (let i = 0; i < tree.subs.length; i++) { + const sub = tree.subs[i]; + const subIsDelimiter = isDelimiter(sub); + if (!subIsDelimiter && lastNodeWasDelimiter) { + flushBlockIntoNewSubs(); + nodesSinceLastFlush = []; + } + lastNodeWasDelimiter = subIsDelimiter; + nodesSinceLastFlush.push(sub); + if (!isBlank(sub)) { + currentBlockIndentation = currentBlockIndentation ?? sub.indentation; + } + } + + // treat the end of node like a block end, and make the virtual block if it wouldn't be a singleton + flushBlockIntoNewSubs(true); + tree.subs = newSubs; + return tree; + }; + return rebuildTree(tree, rebuilder); +} + +/** + * Remove unlabeled virtual nodes which either: + * - Have one or no children + * - Are the only child of their parent + * In either case, it is replaced by their children. + */ +export function flattenVirtual<L>(tree: IndentationTree<L>): IndentationTree<L> { + const rebuilder: Rebuilder<L> = function (tree) { + if (isVirtual(tree) && tree.label === undefined && tree.subs.length <= 1) { + if (tree.subs.length === 0) { + return undefined; + } else { + //tree.subs.length === 1 + return tree.subs[0]; + } + } else if (tree.subs.length === 1 && isVirtual(tree.subs[0]) && tree.subs[0].label === undefined) { + tree.subs = tree.subs[0].subs; + } + return tree; + }; + return rebuildTree(tree, rebuilder); +} + +/** + * Generic labels. + * + * * opener: A line starting with an opening parens, square bracket, or curly brace + * * closer: A line starting with a closing parens, square bracket, or curly brace + */ +const _genericLabelRules = { + opener: /^[[({]/, + closer: /^[\])}]/, +} as const; +const genericLabelRules: LabelRule<'opener' | 'closer'>[] = buildLabelRules(_genericLabelRules); + +const LANGUAGE_SPECIFIC_PARSERS: { [key: string]: (raw: IndentationTree<never>) => IndentationTree<string> } = {}; +/** + * Register a language-specific parser for a language. + * This should normally be called in index.ts. + */ +export function registerLanguageSpecificParser( + language: string, + parser: (raw: IndentationTree<never>) => IndentationTree<string> +): void { + LANGUAGE_SPECIFIC_PARSERS[language] = parser; +} + +export function parseTree(source: string, languageId?: string): IndentationTree<string> { + const raw = parseRaw(source); + const languageSpecificParser = LANGUAGE_SPECIFIC_PARSERS[languageId ?? '']; + if (languageSpecificParser) { + return languageSpecificParser(raw); + } else { + labelLines(raw, genericLabelRules); + const processedTree = combineClosersAndOpeners(raw); + return processedTree; + } +} diff --git a/completions-sample-code/vscode-node/prompt/src/languageMarker.ts b/completions-sample-code/vscode-node/prompt/src/languageMarker.ts new file mode 100644 index 0000000..cf46895 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/languageMarker.ts @@ -0,0 +1,468 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { DocumentInfo } from './prompt'; + +/** + * Interface for writing single-line comments in a given language. + * Does not include the terminal new-line character (i.e. for many languages, + * `end` will just be the empty string). + */ +interface CommentMarker { + start: string; + end: string; +} + +interface ILanguageInfo { + readonly lineComment: CommentMarker; + /** + * if not set, defaults to the language id + */ + readonly markdownLanguageIds?: string[]; +} + +interface ILanguage extends ILanguageInfo { + readonly languageId: string; +} + +/** + * Language files in VSCode: + * https://code.visualstudio.com/docs/languages/identifiers#_known-language-identifiers + * + * Missing below from this list are: + * Diff diff + * Git git-commit and git-rebase + * JSON json + * ShaderLab shaderlab + * Additional to that list are: + * Erlang + * Haskell + * Kotlin + * QL + * Scala + * Verilog + * + * Markdown ids from https://raw.githubusercontent.com/highlightjs/highlight.js/refs/heads/main/SUPPORTED_LANGUAGES.md + * Also refer to [vscode-copilot-chat](https://github.com/microsoft/vscode-copilot-chat/blob/main/src/util/common/languages.ts) + */ +export const languageMarkers: { [language: string]: ILanguageInfo } = { + abap: { + lineComment: { start: '"', end: '' }, + markdownLanguageIds: ['abap', 'sap-abap'], + }, + aspdotnet: { + lineComment: { start: '<%--', end: '--%>' }, + }, + bat: { + lineComment: { start: 'REM', end: '' }, + }, + bibtex: { + lineComment: { start: '%', end: '' }, + markdownLanguageIds: ['bibtex'], + }, + blade: { + lineComment: { start: '#', end: '' }, + }, + BluespecSystemVerilog: { + lineComment: { start: '//', end: '' }, + }, + c: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['c', 'h'], + }, + clojure: { + lineComment: { start: ';', end: '' }, + markdownLanguageIds: ['clojure', 'clj'], + }, + coffeescript: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['coffeescript', 'coffee', 'cson', 'iced'], + }, + cpp: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['cpp', 'hpp', 'cc', 'hh', 'c++', 'h++', 'cxx', 'hxx'], + }, + csharp: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['csharp', 'cs'], + }, + css: { + lineComment: { start: '/*', end: '*/' }, + }, + cuda: { + lineComment: { start: '//', end: '' }, + }, + dart: { + lineComment: { start: '//', end: '' }, + }, + dockerfile: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['dockerfile', 'docker'], + }, + dotenv: { + lineComment: { start: '#', end: '' }, + }, + elixir: { + lineComment: { start: '#', end: '' }, + }, + erb: { + lineComment: { start: '<%#', end: '%>' }, + }, + erlang: { + lineComment: { start: '%', end: '' }, + markdownLanguageIds: ['erlang', 'erl'], + }, + fsharp: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['fsharp', 'fs', 'fsx', 'fsi', 'fsscript'], + }, + go: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['go', 'golang'], + }, + graphql: { + lineComment: { start: '#', end: '' }, + }, + groovy: { + lineComment: { start: '//', end: '' }, + }, + haml: { + lineComment: { start: '-#', end: '' }, + }, + handlebars: { + lineComment: { start: '{{!', end: '}}' }, + markdownLanguageIds: ['handlebars', 'hbs', 'html.hbs', 'html.handlebars'], + }, + haskell: { + lineComment: { start: '--', end: '' }, + markdownLanguageIds: ['haskell', 'hs'], + }, + hlsl: { + lineComment: { start: '//', end: '' }, + }, + html: { + lineComment: { start: '<!--', end: '-->' }, + markdownLanguageIds: ['html', 'xhtml'], + }, + ini: { + lineComment: { start: ';', end: '' }, + }, + java: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['java', 'jsp'], + }, + javascript: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['javascript', 'js'], + }, + javascriptreact: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['jsx'], + }, + jsonc: { + lineComment: { start: '//', end: '' }, + }, + jsx: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['jsx'], + }, + julia: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['julia', 'jl'], + }, + kotlin: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['kotlin', 'kt'], + }, + latex: { + lineComment: { start: '%', end: '' }, + markdownLanguageIds: ['tex'], + }, + legend: { + lineComment: { start: '//', end: '' }, + }, + less: { + lineComment: { start: '//', end: '' }, + }, + lua: { + lineComment: { start: '--', end: '' }, + markdownLanguageIds: ['lua', 'pluto'], + }, + makefile: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['makefile', 'mk', 'mak', 'make'], + }, + markdown: { + lineComment: { start: '[]: #', end: '' }, + markdownLanguageIds: ['markdown', 'md', 'mkdown', 'mkd'], + }, + 'objective-c': { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['objectivec', 'mm', 'objc', 'obj-c'], + }, + 'objective-cpp': { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['objectivec++', 'objc+'], + }, + perl: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['perl', 'pl', 'pm'], + }, + php: { + lineComment: { start: '//', end: '' }, + }, + powershell: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['powershell', 'ps', 'ps1'], + }, + pug: { + lineComment: { start: '//', end: '' }, + }, + python: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['python', 'py', 'gyp'], + }, + ql: { + lineComment: { start: '//', end: '' }, + }, // QL is a query language for CodeQL + r: { + lineComment: { start: '#', end: '' }, + }, + razor: { + lineComment: { start: '<!--', end: '-->' }, + markdownLanguageIds: ['cshtml', 'razor', 'razor-cshtml'], + }, + ruby: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['ruby', 'rb', 'gemspec', 'podspec', 'thor', 'irb'], + }, + rust: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['rust', 'rs'], + }, + sass: { + lineComment: { start: '//', end: '' }, + }, + scala: { + lineComment: { start: '//', end: '' }, + }, + scss: { + lineComment: { start: '//', end: '' }, + }, + shellscript: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['bash', 'sh', 'zsh'], + }, + slang: { + lineComment: { start: '//', end: '' }, + }, + slim: { + lineComment: { start: '/', end: '' }, + }, + solidity: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['solidity', 'sol'], + }, + sql: { + lineComment: { start: '--', end: '' }, + }, + stylus: { + lineComment: { start: '//', end: '' }, + }, + svelte: { + lineComment: { start: '<!--', end: '-->' }, + }, + swift: { + lineComment: { start: '//', end: '' }, + }, + systemverilog: { + lineComment: { start: '//', end: '' }, + }, + terraform: { + lineComment: { start: '#', end: '' }, + }, + tex: { + lineComment: { start: '%', end: '' }, + }, + typescript: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['typescript', 'ts'], + }, + typescriptreact: { + lineComment: { start: '//', end: '' }, + markdownLanguageIds: ['tsx'], + }, + vb: { + lineComment: { start: `'`, end: '' }, + markdownLanguageIds: ['vb', 'vbscript'], + }, + verilog: { + lineComment: { start: '//', end: '' }, + }, + 'vue-html': { + lineComment: { start: '<!--', end: '-->' }, + }, + vue: { + lineComment: { start: '//', end: '' }, + }, + xml: { + lineComment: { start: '<!--', end: '-->' }, + }, + xsl: { + lineComment: { start: '<!--', end: '-->' }, + }, + yaml: { + lineComment: { start: '#', end: '' }, + markdownLanguageIds: ['yaml', 'yml'], + }, +}; + +const mdLanguageIdToLanguageId: { [markdownLanguageId: string]: string } = {}; +for (const [languageId, info] of Object.entries(languageMarkers)) { + if (info.markdownLanguageIds) { + for (const mdLanguageId of info.markdownLanguageIds) { + mdLanguageIdToLanguageId[mdLanguageId] = languageId; + } + } else { + mdLanguageIdToLanguageId[languageId] = languageId; + } +} + +export function mdCodeBlockLangToLanguageId(mdLanguageId: string): string | undefined { + return mdLanguageIdToLanguageId[mdLanguageId]; +} + +const defaultCommentMarker: CommentMarker = { start: '//', end: '' }; + +const dontAddLanguageMarker: string[] = [ + 'php', // We don't know if the file starts with `<?php` or not + 'plaintext', // Doesn't admit comments +]; + +// prettier-ignore +const shebangLines: { [language: string]: string } = { + 'html': '<!DOCTYPE html>', + 'python': '#!/usr/bin/env python3', + 'ruby': '#!/usr/bin/env ruby', + 'shellscript': '#!/bin/sh', + 'yaml': '# YAML data' +}; + +/** + * Determine if a line is a shebang line for a known language + * @param line The line to check + * @returns The language if it is a known shebang line, otherwise undefined + */ +export function isShebangLine(line: string): boolean { + return Object.values(shebangLines).includes(line.trim()); +} + +/** + * Best-effort determining whether the top of the source already contains a + * discernible language marker, in particular a shebang line + * @param languageId The string name of the language + * @returns True iff we determined a recognisable language marker + */ +// prettier-ignore +export function hasLanguageMarker({ source }: DocumentInfo): boolean { + return source.startsWith('#!') || source.startsWith('<!DOCTYPE'); +} + +/** + * Comment a single line of text in a given language. + * E.g. for python, turn "hello there" into "# hello there" + * + * Note: This will not behave as you expect if `text` has multiple lines. In + * that case, use {@link commentBlockAsSingles} instead. + */ +export function comment(text: string, languageId: string) { + const markers = languageMarkers[languageId] ? languageMarkers[languageId].lineComment : defaultCommentMarker; + if (markers) { + const end = markers.end === '' ? '' : ' ' + markers.end; + return `${markers.start} ${text}${end}`; + } + return ''; +} + +/** + * Comment a block of text using single-line comments. + * + * The returned comment block will have a trailing newline exactly when the + * input does. + */ +export function commentBlockAsSingles(text: string, languageId: string) { + if (text === '') { + // Avoid spewing out a long list of blank lines + return ''; + } + const trailingNewline = text.endsWith('\n'); + const lines = (trailingNewline ? text.slice(0, -1) : text).split('\n'); + const commented = lines.map(line => comment(line, languageId)).join('\n'); + return trailingNewline ? commented + '\n' : commented; +} + +/** + * Return a one-line comment or text which describes the language of a + * document, e.g. a shebang line or a comment. + * + * @param doc The document we want the marker for. + * @returns A one-line string that describes the language. + */ +export function getLanguageMarker(doc: DocumentInfo): string { + const { languageId } = doc; + if (dontAddLanguageMarker.indexOf(languageId) === -1 && !hasLanguageMarker(doc)) { + if (languageId in shebangLines) { + return shebangLines[languageId]; + } else { + return `Language: ${languageId}`; + } + } + return ''; +} + +/** + * Return a one-line comment containing the relative path of the document, if known. + * + * @param doc The document we want the marker for. + * @param defaultCommentMarker The comment marker to use if the language does not have one. + * @returns A one-line comment that contains the relative path of the document. + */ +export function getPathMarker(doc: DocumentInfo): string { + if (doc.relativePath) { + return `Path: ${doc.relativePath}`; + } + return ''; +} + +/** + * Appends a new line to a string if it does not already end with one. + * + * @param str String to append + * + * @returns A string with a new line escape character at the end. + */ +export function newLineEnded(str: string): string { + return str === '' || str.endsWith('\n') ? str : str + '\n'; +} + +/** + * Retrieves the language for a given language identifier. + * + * @param languageId - The identifier of the language. If undefined, defaults to 'plaintext'. + * @returns The language associated with the specified language identifier. + */ +export function getLanguage(languageId: string | undefined): ILanguage { + if (typeof languageId === 'string') { + return _getLanguage(languageId); + } + return _getLanguage('plaintext'); +} + +function _getLanguage(languageId: string): ILanguage { + if (languageMarkers[languageId] !== undefined) { + return { languageId, ...languageMarkers[languageId] }; + } else { + return { languageId, lineComment: { start: '//', end: '' } }; + } +} diff --git a/completions-sample-code/vscode-node/prompt/src/parse.ts b/completions-sample-code/vscode-node/prompt/src/parse.ts new file mode 100644 index 0000000..b25799b --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/parse.ts @@ -0,0 +1,172 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import Parser from 'web-tree-sitter'; +import { CopilotPromptLoadFailure } from './error'; +import { locateFile, readFile } from './fileLoader'; + +export enum WASMLanguage { + Python = 'python', + JavaScript = 'javascript', + TypeScript = 'typescript', + TSX = 'tsx', + Go = 'go', + Ruby = 'ruby', + CSharp = 'c-sharp', + Java = 'java', + Php = 'php', + Cpp = 'cpp', +} + +const languageIdToWasmLanguageMapping: { [language: string]: WASMLanguage } = { + python: WASMLanguage.Python, + javascript: WASMLanguage.JavaScript, + javascriptreact: WASMLanguage.JavaScript, + jsx: WASMLanguage.JavaScript, + typescript: WASMLanguage.TypeScript, + typescriptreact: WASMLanguage.TSX, + go: WASMLanguage.Go, + ruby: WASMLanguage.Ruby, + csharp: WASMLanguage.CSharp, + java: WASMLanguage.Java, + php: WASMLanguage.Php, + c: WASMLanguage.Cpp, + cpp: WASMLanguage.Cpp, +}; + +export function isSupportedLanguageId(languageId: string): boolean { + // Temporarily disable C# support until the tree-sitter parser for it is + // fully spec-ed. + return ( + languageId in languageIdToWasmLanguageMapping && + languageId !== 'csharp' && + languageId !== 'java' && + languageId !== 'php' && + languageId !== 'c' && + languageId !== 'cpp' + ); +} + +export function languageIdToWasmLanguage(languageId: string): WASMLanguage { + if (!(languageId in languageIdToWasmLanguageMapping)) { + throw new Error(`Unrecognized language: ${languageId}`); + } + return languageIdToWasmLanguageMapping[languageId]; +} + +const languageLoadPromises = new Map<WASMLanguage, Promise<Parser.Language>>(); + +async function loadWasmLanguage(language: WASMLanguage): Promise<Parser.Language> { + // construct a path that works both for the TypeScript source, which lives under `/src`, and for + // the transpiled JavaScript, which lives under `/dist` + let wasmBytes; + try { + wasmBytes = await readFile(`tree-sitter-${language}.wasm`); + } catch (e: unknown) { + if (e instanceof Error && 'code' in e && typeof e.code === 'string' && e.name === 'Error') { + throw new CopilotPromptLoadFailure(`Could not load tree-sitter-${language}.wasm`, e); + } + throw e; + } + return Parser.Language.load(wasmBytes); +} + +export function getLanguage(language: string): Promise<Parser.Language> { + const wasmLanguage = languageIdToWasmLanguage(language); + + if (!languageLoadPromises.has(wasmLanguage)) { + // IMPORTANT: This function does not have an async signature to prevent interleaved execution + // that can cause duplicate loading of the same language during yields/awaits prior to them + // being added to the cache. + const loadedLang = loadWasmLanguage(wasmLanguage); + languageLoadPromises.set(wasmLanguage, loadedLang); + } + + return languageLoadPromises.get(wasmLanguage)!; +} + +class WrappedError extends Error { + constructor(message: string, cause: unknown) { + super(message, { cause }); + } +} + +// This method returns a tree that the user needs to call `.delete()` before going out of scope. +export async function parseTreeSitter(language: string, source: string): Promise<Parser.Tree> { + return (await parseTreeSitterIncludingVersion(language, source))[0]; +} + +// This method returns a tree that the user needs to call `.delete()` before going out of scope. +export async function parseTreeSitterIncludingVersion(language: string, source: string): Promise<[Parser.Tree, number]> { + // `Parser.init` needs to be called before `new Parser()` below + await Parser.init({ + locateFile: (filename: string) => locateFile(filename), + }); + let parser; + try { + parser = new Parser(); + } catch (e: unknown) { + if ( + e && + typeof e === 'object' && + 'message' in e && + typeof e.message === 'string' && + e.message.includes('table index is out of bounds') + ) { + throw new WrappedError(`Could not init Parse for language <${language}>`, e); + } + throw e; + } + const treeSitterLanguage = await getLanguage(language); + parser.setLanguage(treeSitterLanguage); + const parsedTree = parser.parse(source); + + // Need to delete parser objects directly + parser.delete(); + return [parsedTree, treeSitterLanguage.version]; +} + +export function getBlockCloseToken(language: string): string | null { + const wasmLanguage = languageIdToWasmLanguage(language); + switch (wasmLanguage) { + case WASMLanguage.Python: + return null; + case WASMLanguage.JavaScript: + case WASMLanguage.TypeScript: + case WASMLanguage.TSX: + case WASMLanguage.Go: + case WASMLanguage.CSharp: + case WASMLanguage.Java: + case WASMLanguage.Php: + case WASMLanguage.Cpp: + return '}'; + case WASMLanguage.Ruby: + return 'end'; + } +} + +function innerQuery(queries: [string, Parser.Query?][], root: Parser.SyntaxNode): Parser.QueryMatch[] { + const matches = []; + for (const query of queries) { + // parse and cache query if this is the first time we've used it + if (!query[1]) { + const lang = root.tree.getLanguage(); + // cache parsed query object + query[1] = lang.query(query[0]); + } + matches.push(...query[1].matches(root)); + } + return matches; +} + +const docstringQuery: [string, Parser.Query?] = [ + `[ + (class_definition (block (expression_statement (string)))) + (function_definition (block (expression_statement (string)))) +]`, +]; + +export function queryPythonIsDocstring(blockNode: Parser.SyntaxNode): boolean { + return innerQuery([docstringQuery], blockNode).length === 1; +} diff --git a/completions-sample-code/vscode-node/prompt/src/parseBlock.ts b/completions-sample-code/vscode-node/prompt/src/parseBlock.ts new file mode 100644 index 0000000..b1b2fa9 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/parseBlock.ts @@ -0,0 +1,975 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as Parser from 'web-tree-sitter'; +import { + WASMLanguage, + isSupportedLanguageId, + languageIdToWasmLanguage, + parseTreeSitter, + parseTreeSitterIncludingVersion, + queryPythonIsDocstring, +} from './parse'; + +interface BlockParser { + isEmptyBlockStart: (text: string, offset: number) => Promise<boolean>; + + /** + * Given a document prefix, offset, and a proposed completion, determines how much of the + * completion to keep in order to "finish" the following block when the completion is appended + * to the document prefix. + * + * If there is no such block, or the completion doesn't close the block, returns undefined. + */ + isBlockBodyFinished: (prefix: string, completion: string, offset: number) => Promise<number | undefined>; + + /** + * Given a document text and offset, determines the beginning of current matching node. + * + * If there is no such block, returns undefined. + */ + getNodeStart: (text: string, offset: number) => Promise<number | undefined>; +} + +abstract class BaseBlockParser implements BlockParser { + abstract isEmptyBlockStart(text: string, offset: number): Promise<boolean>; + + constructor( + protected readonly languageId: string, + protected readonly nodeMatch: { [parent: string]: string }, + /** + * A map from node types that have a block or an statement as a child + * to the field label of the child node that is a block or statement. + * For example, an if statement in a braced language. + */ + protected readonly nodeTypesWithBlockOrStmtChild: Map<string, string> + ) { } + + protected async getNodeMatchAtPosition<T>( + text: string, + offset: number, + cb: (nd: Parser.SyntaxNode) => T + ): Promise<T | undefined> { + const tree = await parseTreeSitter(this.languageId, text); + try { + // TODO:(hponde) It seems that we have an issue if it's at the end of the block: + // https://github.com/tree-sitter/tree-sitter/issues/407 + const nodeAtPos = tree.rootNode.descendantForIndex(offset); + + let nodeToComplete: Parser.SyntaxNode | null = nodeAtPos; + + // find target element by looking at parent of cursor node + // don't stop at node types that may have a block child, but don't actually in this + // parse tree + while (nodeToComplete) { + const blockNodeType = this.nodeMatch[nodeToComplete.type]; + if (blockNodeType) { + if (!this.nodeTypesWithBlockOrStmtChild.has(nodeToComplete.type)) { + break; + } + + const fieldLabel = this.nodeTypesWithBlockOrStmtChild.get(nodeToComplete.type)!; + const childToCheck = + fieldLabel === '' + ? nodeToComplete.namedChildren[0] + : nodeToComplete.childForFieldName(fieldLabel); + if (childToCheck?.type === blockNodeType) { + break; + } + } + + nodeToComplete = nodeToComplete.parent; + } + if (!nodeToComplete) { + // No nodes we're interested in + return; + } + return cb(nodeToComplete); + } finally { + tree.delete(); + } + } + + protected getNextBlockAtPosition<T>( + text: string, + offset: number, + cb: (nd: Parser.SyntaxNode) => T + ): Promise<T | undefined> { + return this.getNodeMatchAtPosition(text, offset, nodeToComplete => { + // FIXME: childForFieldName always returns null + // const block = nodeToComplete.childForFieldName(fieldToComplete); + // Instead, find child nodes of the langauge's nodeMatch type for + // nodeToComplete. + // Look in reverse order, in case of nodes with multiple blocks defined, + // such as try/catch/finally. + let block = nodeToComplete.children.reverse().find(x => x.type === this.nodeMatch[nodeToComplete.type]); + if (!block) { + // child of matching type isn't defined yet + return; + } + + if (this.languageId === 'python' && block.parent) { + // handle empty block's parent being the colon (!) + const parent = block.parent.type === ':' ? block.parent.parent : block.parent; + + // tree-sitter handles comments in a weird way, so we need to + // consume them. + let nextComment = parent?.nextSibling; + + while (nextComment && nextComment.type === 'comment') { + // next comment is inline at the end of the block + // see issue: https://github.com/tree-sitter/tree-sitter-python/issues/113 + const commentInline = + nextComment.startPosition.row === block.endPosition.row && + nextComment.startPosition.column >= block.endPosition.column; + + // next comment is on subsequent line and indented > parent's indentation + // see issue: https://github.com/tree-sitter/tree-sitter-python/issues/112 + const commentAtEnd = + nextComment.startPosition.row > parent!.endPosition.row && + nextComment.startPosition.column > parent!.startPosition.column; + + if (commentInline || commentAtEnd) { + block = nextComment; + nextComment = nextComment.nextSibling; + } else { + break; + } + } + } + + if (block.endIndex >= block.tree.rootNode.endIndex - 1 && (block.hasError || block.parent!.hasError)) { + // TODO:(hponde) improve this logic + // block is the whole document, and has errors, most likely doc has + // preceding errors. + return; + } + + // Return first block if not empty + return cb(block); + }); + } + + async isBlockBodyFinished(prefix: string, completion: string, offset: number): Promise<number | undefined> { + const solution = (prefix + completion).trimEnd(); + const endIndex = await this.getNextBlockAtPosition(solution, offset, block => block.endIndex); + if (endIndex === undefined) { + // no block, not finished yet + return; + } + if (endIndex < solution.length) { + // descendant block is finished, stop at end of block + const lengthOfBlock = endIndex - prefix.length; + return lengthOfBlock > 0 ? lengthOfBlock : undefined; + } + } + + getNodeStart(text: string, offset: number): Promise<number | undefined> { + const solution = text.trimEnd(); + return this.getNodeMatchAtPosition(solution, offset, block => block.startIndex); + } +} + +class RegexBasedBlockParser extends BaseBlockParser { + constructor( + languageId: string, + protected readonly blockEmptyMatch: string, + private readonly lineMatch: RegExp, + nodeMatch: { [parent: string]: string }, + nodeTypesWithBlockOrStmtChild: Map<string, string> + ) { + super(languageId, nodeMatch, nodeTypesWithBlockOrStmtChild); + } + + private isBlockStart(line: string): boolean { + return this.lineMatch.test(line.trimStart()); + } + + private async isBlockBodyEmpty(text: string, offset: number): Promise<boolean> { + const res = await this.getNextBlockAtPosition(text, offset, block => { + // strip whitespace and compare with language-defined empty block + // Note that for Ruby, `block` is the closing `end` token, while for other + // languages it is the whole block, so we consider the text from the earlier of + // block.startIndex and offset, all the way up to block.endIndex. + if (block.startIndex < offset) { offset = block.startIndex; } + const blockText = text.substring(offset, block.endIndex).trim(); + if (blockText === '' || blockText.replace(/\s/g, '') === this.blockEmptyMatch) { + // block is empty + return true; + } + return false; + }); + return res === undefined || res; + } + + async isEmptyBlockStart(text: string, offset: number): Promise<boolean> { + offset = rewindToNearestNonWs(text, offset); + return this.isBlockStart(getLineAtOffset(text, offset)) && this.isBlockBodyEmpty(text, offset); + } +} + +function getLineAtOffset(text: string, offset: number): string { + const prevNewline = text.lastIndexOf('\n', offset - 1); + let nextNewline = text.indexOf('\n', offset); + if (nextNewline < 0) { + nextNewline = text.length; + } + return text.slice(prevNewline + 1, nextNewline); +} + +/** + * Returns the cursor position immediately after the nearest non-whitespace + * character. If every character before offset is whitespace, returns 0. + */ +function rewindToNearestNonWs(text: string, offset: number): number { + let result = offset; + while (result > 0 && /\s/.test(text.charAt(result - 1))) { + result--; + } + return result; +} + +/** + * If `nd` is only preceded by whitespace on the line where it starts, return that whitespace; + * otherwise, return undefined. The parameter `source` is the source text from which `nd` was + * parsed. + */ +function indent(nd: Parser.SyntaxNode, source: string): string | undefined { + const startIndex = nd.startIndex; + const lineStart = nd.startIndex - nd.startPosition.column; + const prefix = source.substring(lineStart, startIndex); + if (/^\s*$/.test(prefix)) { + return prefix; + } + return undefined; +} + +/** + * Check if `snd` is "outdented" with respect to `fst`, that is, it starts on a later line, and + * its indentation is no greater than that of `fst`. + */ +function outdented(fst: Parser.SyntaxNode, snd: Parser.SyntaxNode, source: string): boolean { + if (snd.startPosition.row <= fst.startPosition.row) { + return false; + } + const fstIndent = indent(fst, source); + const sndIndent = indent(snd, source); + return fstIndent !== undefined && sndIndent !== undefined && fstIndent.startsWith(sndIndent); +} + +class TreeSitterBasedBlockParser extends BaseBlockParser { + constructor( + languageId: string, + nodeMatch: { [parent: string]: string }, + nodeTypesWithBlockOrStmtChild: Map<string, string>, + private readonly startKeywords: string[], + private readonly blockNodeType: string, + /** + * The langauge-specific node type of an empty statement, that is, + * a statement with no text except possibly the statement terminator. + * For example, `;` is an empty statement in a braced language, but + * `pass` is not in Python. + */ + private readonly emptyStatementType: string | null, + private readonly curlyBraceLanguage: boolean + ) { + super(languageId, nodeMatch, nodeTypesWithBlockOrStmtChild); + } + + private isBlockEmpty(block: Parser.SyntaxNode, offset: number): boolean { + let trimmed = block.text.trim(); + + if (this.curlyBraceLanguage) { + if (trimmed.startsWith('{')) { + trimmed = trimmed.slice(1); + } + if (trimmed.endsWith('}')) { + trimmed = trimmed.slice(0, -1); + } + trimmed = trimmed.trim(); + } + + if (trimmed.length === 0) { + return true; + } + + // Python: Consider a block that contains only a docstring empty. + if ( + this.languageId === 'python' && + (block.parent?.type === 'class_definition' || block.parent?.type === 'function_definition') && + block.children.length === 1 && + queryPythonIsDocstring(block.parent) + ) { + return true; + } + + return false; + } + + async isEmptyBlockStart(text: string, offset: number): Promise<boolean> { + if (offset > text.length) { + throw new RangeError('Invalid offset'); + } + + // Ensure that the cursor is at the end of a line, ignoring trailing whitespace. + for (let i = offset; i < text.length; i++) { + if (text.charAt(i) === '\n') { + break; + } else if (/\S/.test(text.charAt(i))) { + return false; + } + } + + // This lets e.g. "def foo():\nรขโ€“ห†" give a multiline suggestion. + offset = rewindToNearestNonWs(text, offset); + + const [tree, version] = await parseTreeSitterIncludingVersion(this.languageId, text); + try { + // offset here is the cursor position immediately after a whitespace + // character, but tree-sitter expects the index of the node to search for. + // Therefore we adjust the offset when we call into tree-sitter. + const nodeAtPos = tree.rootNode.descendantForIndex(offset - 1); + if (nodeAtPos === null) { + return false; + } + + // Because of rewinding to the previous non-whitespace character, nodeAtPos may be + // "}". That's not a good place to show multline ghost text. + if (this.curlyBraceLanguage && nodeAtPos.type === '}') { + return false; + } + + // JS/TS: half open, empty blocks are sometimes parsed as objects + if ( + (this.languageId === 'javascript' || this.languageId === 'typescript') && + nodeAtPos.parent && + nodeAtPos.parent.type === 'object' && + nodeAtPos.parent.text.trim() === '{' + ) { + return true; + } + + // TS: a function_signature/method_signature is a prefix of a + // function_declaration/method_declaration, so if nodeAtPos is a descendant of one of + // those node types and the signature looks incomplete, return true + if (this.languageId === 'typescript') { + let currNode = nodeAtPos; + while (currNode.parent) { + if (currNode.type === 'function_signature' || currNode.type === 'method_signature') { + // if the next node is outdented, the signature is probably incomplete and + // TreeSitter may just have done some fanciful error correction, so we'll + // assume that this is really meant to be an incomplete function + const next = nodeAtPos.nextSibling; + if (next && currNode.hasError && outdented(currNode, next, text)) { + return true; + } + + // if, on the other hand, there is a semicolon, then the signature is + // probably complete, and we should not show a multiline suggestion + const semicolon = currNode.children.find(c => c.type === ';'); + return !semicolon && currNode.endIndex <= offset; + } + currNode = currNode.parent; + } + } + + // Ignoring special cases, there are three situations when we want to return true: + // + // 1. nodeAtPos is in a block or a descendant of a block, the parent of the block is one of the node types + // in this.nodeMatch, and the block is empty. + // 2. nodeAtPos is somewhere below an ERROR node, and that ERROR node has an anonymous child + // matching one of the keywords we care about. If that ERROR node also has a block child, the + // block must be empty. + // 3. nodeAtPos is somewhere below a node type that we know can contain a block, and the block is either + // not present or empty. + + let errorNode = null; + let blockNode = null; + let blockParentNode = null; + let currNode: Parser.SyntaxNode | null = nodeAtPos; + while (currNode !== null) { + if (currNode.type === this.blockNodeType) { + blockNode = currNode; + break; + } + if (this.nodeMatch[currNode.type]) { + blockParentNode = currNode; + break; + } + if (currNode.type === 'ERROR') { + errorNode = currNode; + break; + } + currNode = currNode.parent; + } + if (blockNode !== null) { + if (!blockNode.parent || !this.nodeMatch[blockNode.parent.type]) { + return false; + } + + // Python: hack for unclosed docstrings. There's no rhyme or reason to how the actual + // docstring comments are parsed, but overall the parse tree looks like: + // function_definition + // - def + // - identifier + // - parameters + // - : + // - ERROR with text that starts with """ or ''' + // - block + // - junk + // + // We do best effort here to detect that we're in an unclosed docstring and return true. + // Note that this won't work (we won't give a multline suggestion) if the docstring uses single + // quotes, which is allowed by the language standard but not idiomatic (see PEP 257, + // Docstring Conventions). + if (this.languageId === 'python') { + const prevSibling = blockNode.previousSibling; + if ( + prevSibling !== null && + prevSibling.hasError && + (prevSibling.text.startsWith('"""') || prevSibling.text.startsWith(`'''`)) + ) { + return true; + } + } + + return this.isBlockEmpty(blockNode, offset); + } + if (errorNode !== null) { + // TS: In a module such as "module 'foo' {" or internal_module such as "namespace 'foo' {" + // the open brace is parsed as an error node, like so: + // - expression_statement + // - [internal_]module + // - string + // - ERROR + if ( + errorNode.previousSibling?.type === 'module' || + errorNode.previousSibling?.type === 'internal_module' || + errorNode.previousSibling?.type === 'def' + ) { + return true; + } + + // @dbaeumer The way how unfinished docstrings are handled changed in version 14 for Python. + if (this.languageId === 'python' && version >= 14) { + // In version 14 and later, we need to account for the possibility of + // an unfinished docstring being represented as an ERROR node. + if (errorNode.hasError && (errorNode.text.startsWith('"') || errorNode.text.startsWith(`'`))) { + const parentType = errorNode.parent?.type; + if ( + parentType === 'function_definition' || + parentType === 'class_definition' || + parentType === 'module' + ) { + return true; + } + } + } + + // Search in reverse order so we get the latest block or keyword node. + const children = [...errorNode.children].reverse(); + const keyword = children.find(child => this.startKeywords.includes(child.type)); + let block = children.find(child => child.type === this.blockNodeType); + + if (keyword) { + switch (this.languageId) { + case 'python': { + // Python: try-except-finally + // If the cursor is in either "except" or "finally," but the try-except-finally isn't finished, + // nodeAtPos will be parsed as an identifier. If > 4 characters of "except" or "finally" have been + // typed, it will be parsed as: + // ERROR + // - try + // - : + // - ERROR + // - block + // - expression_statement + // - identifier + // + // In this case, we have to special-case finding the right block to check whether it's empty. + if (keyword.type === 'try' && nodeAtPos.type === 'identifier' && nodeAtPos.text.length > 4) { + block = children + .find(child => child.hasError) + ?.children.find(child => child.type === 'block'); + } + + // Python: sometimes nodes that are morally part of a block are parsed as statements + // that are all children of an ERROR node. Detect this by looking for ":" and inspecting + // its nextSibling. Skip over ":" inside parentheses because those could be part of a + // typed parameter. + let colonNode; + let parenCount = 0; + for (const child of errorNode.children) { + if (child.type === ':' && parenCount === 0) { + colonNode = child; + break; + } + if (child.type === '(') { + parenCount += 1; + } + if (child.type === ')') { + parenCount -= 1; + } + } + if (colonNode && keyword.endIndex <= colonNode.startIndex && colonNode.nextSibling) { + // horrible hack to handle unfinished docstrings :( + if (keyword.type === 'def') { + const sibling = colonNode.nextSibling; + if (sibling.type === '"' || sibling.type === `'`) { + return true; + } + if (sibling.type === 'ERROR' && (sibling.text === '"""' || sibling.text === `'''`)) { + return true; + } + } + return false; + } + + break; + } + case 'javascript': { + // JS: method definition within a class, e.g. "class C { foo()" + if (keyword.type === 'class') { + if (version <= 13) { + const formalParameters = children.find(child => child.type === 'formal_parameters'); + if (formalParameters) { + return true; + } + } else { + const children = errorNode.children; + for (let i = 0; i < children.length; i++) { + const child = children[i]; + if (child.type === 'formal_parameters') { + return ( + i + 1 === children.length || + (children[i + 1]?.type === '{' && i + 2 === children.length) + ); + } + } + } + } + + // JS: Don't mistake a half-open curly brace after a keyword under an error node for an empty + // block. If it has a nextSibling, then it's not empty. e.g. in "do {\n\t;รขโ€“ห†", the ";" is an + // empty_statement and the nextSibling of the "{". + const leftCurlyBrace = children.find(child => child.type === '{'); + if ( + leftCurlyBrace && + leftCurlyBrace.startIndex > keyword.endIndex && + leftCurlyBrace.nextSibling !== null + ) { + return false; + } + + // JS: do-while: don't give a multline suggestion after the "while" keyword + const doNode = children.find(child => child.type === 'do'); + if (doNode && keyword.type === 'while') { + return false; + } + + // JS: In an arrow function, if there is a next sibling of the arrow and it's not an open brace, we're not in a + // block context and we should return false. + if (keyword.type === '=>' && keyword.nextSibling && keyword.nextSibling.type !== '{') { + return false; + } + + break; + } + case 'typescript': { + // TS: Don't mistake a half-open curly brace after a keyword under an error node for an empty + // block. If it has a nextSibling, then it's not empty. e.g. in "do {\n\t;รขโ€“ห†", the ";" is an + // empty_statement and the nextSibling of the "{". + const leftCurlyBrace = children.find(child => child.type === '{'); + if ( + leftCurlyBrace && + leftCurlyBrace.startIndex > keyword.endIndex && + leftCurlyBrace.nextSibling !== null + ) { + return false; + } + + // TS: do-while: don't give a multline suggestion after the "while" keyword + const doNode = children.find(child => child.type === 'do'); + if (doNode && keyword.type === 'while') { + return false; + } + + // TS: In an arrow function, if there is a next sibling of the arrow and it's not an open brace, we're not in a + // block context and we should return false. + if (keyword.type === '=>' && keyword.nextSibling && keyword.nextSibling.type !== '{') { + return false; + } + + break; + } + } + + if (block && block.startIndex > keyword.endIndex) { + return this.isBlockEmpty(block, offset); + } + return true; + } + } + if (blockParentNode !== null) { + const expectedType = this.nodeMatch[blockParentNode.type]; + const block = blockParentNode.children + .slice() + .reverse() + .find(x => x.type === expectedType); + if (!block) { + // Some node types have a child that is either a block or a statement, e.g. "if (foo)". + // If the user has started typing a non-block statement, then this is not the start of an + // empty block. + if (this.nodeTypesWithBlockOrStmtChild.has(blockParentNode.type)) { + const fieldLabel = this.nodeTypesWithBlockOrStmtChild.get(blockParentNode.type)!; + const child = + fieldLabel === '' + ? blockParentNode.children[0] + : blockParentNode.childForFieldName(fieldLabel); + if (child && child.type !== this.blockNodeType && child.type !== this.emptyStatementType) { + return false; + } + } + + return true; + } else { + return this.isBlockEmpty(block, offset); + } + } + + return false; + } finally { + tree.delete(); + } + } +} + +const wasmLanguageToBlockParser: { [languageId in WASMLanguage]: BlockParser } = { + python: new TreeSitterBasedBlockParser( + /* languageId */ 'python', + /* nodeMatch */ { + // Generated with script/tree-sitter-super-types tree-sitter-python block + class_definition: 'block', + elif_clause: 'block', + else_clause: 'block', + except_clause: 'block', + finally_clause: 'block', + for_statement: 'block', + function_definition: 'block', + if_statement: 'block', + try_statement: 'block', + while_statement: 'block', + with_statement: 'block', + }, + /* nodeTypesWithBlockOrStmtChild */ new Map(), + /* startKeywords */['def', 'class', 'if', 'elif', 'else', 'for', 'while', 'try', 'except', 'finally', 'with'], + /* blockNodeType */ 'block', + /* emptyStatementType */ null, + /* curlyBraceLanguage */ false + ), + javascript: new TreeSitterBasedBlockParser( + /* languageId */ 'javascript', + /* nodeMatch */ { + // Generated with script/tree-sitter-super-types tree-sitter-javascript statement_block + arrow_function: 'statement_block', + catch_clause: 'statement_block', + do_statement: 'statement_block', + else_clause: 'statement_block', + finally_clause: 'statement_block', + for_in_statement: 'statement_block', + for_statement: 'statement_block', + function: 'statement_block', + function_expression: 'statement_block', + function_declaration: 'statement_block', + generator_function: 'statement_block', + generator_function_declaration: 'statement_block', + if_statement: 'statement_block', + method_definition: 'statement_block', + try_statement: 'statement_block', + while_statement: 'statement_block', + with_statement: 'statement_block', + // Generated with script/tree-sitter-super-types tree-sitter-javascript class_body + class: 'class_body', + class_declaration: 'class_body', + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + ['arrow_function', 'body'], + ['do_statement', 'body'], + ['else_clause', ''], + ['for_in_statement', 'body'], + ['for_statement', 'body'], + ['if_statement', 'consequence'], + ['while_statement', 'body'], + ['with_statement', 'body'], + ]), + /* startKeywords */[ + '=>', + 'try', + 'catch', + 'finally', + 'do', + 'for', + 'if', + 'else', + 'while', + 'with', + 'function', + 'function*', + 'class', + ], + /* blockNodeType */ 'statement_block', + /* emptyStatementType */ 'empty_statement', + /* curlyBraceLanguage */ true + ), + typescript: new TreeSitterBasedBlockParser( + /* languageId */ 'typescript', + /* nodeMatch */ { + // Generated with script/tree-sitter-super-types tree-sitter-typescript/typescript statement_block + ambient_declaration: 'statement_block', + arrow_function: 'statement_block', + catch_clause: 'statement_block', + do_statement: 'statement_block', + else_clause: 'statement_block', + finally_clause: 'statement_block', + for_in_statement: 'statement_block', + for_statement: 'statement_block', + function: 'statement_block', + function_expression: 'statement_block', + function_declaration: 'statement_block', + generator_function: 'statement_block', + generator_function_declaration: 'statement_block', + if_statement: 'statement_block', + internal_module: 'statement_block', + method_definition: 'statement_block', + module: 'statement_block', + try_statement: 'statement_block', + while_statement: 'statement_block', + // Generated with script/tree-sitter-super-types tree-sitter-typescript/typescript class_body + abstract_class_declaration: 'class_body', + class: 'class_body', + class_declaration: 'class_body', + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + ['arrow_function', 'body'], + ['do_statement', 'body'], + ['else_clause', ''], + ['for_in_statement', 'body'], + ['for_statement', 'body'], + ['if_statement', 'consequence'], + ['while_statement', 'body'], + ['with_statement', 'body'], + ]), + /* startKeywords */[ + 'declare', + '=>', + 'try', + 'catch', + 'finally', + 'do', + 'for', + 'if', + 'else', + 'while', + 'with', + 'function', + 'function*', + 'class', + ], + /* blockNodeType */ 'statement_block', + /* emptyStatementType */ 'empty_statement', + /* curlyBraceLanguage */ true + ), + tsx: new TreeSitterBasedBlockParser( + /* languageId */ 'typescriptreact', + /* nodeMatch */ { + // Generated with script/tree-sitter-super-types tree-sitter-typescript/typescript statement_block + ambient_declaration: 'statement_block', + arrow_function: 'statement_block', + catch_clause: 'statement_block', + do_statement: 'statement_block', + else_clause: 'statement_block', + finally_clause: 'statement_block', + for_in_statement: 'statement_block', + for_statement: 'statement_block', + function: 'statement_block', + function_expression: 'statement_block', + function_declaration: 'statement_block', + generator_function: 'statement_block', + generator_function_declaration: 'statement_block', + if_statement: 'statement_block', + internal_module: 'statement_block', + method_definition: 'statement_block', + module: 'statement_block', + try_statement: 'statement_block', + while_statement: 'statement_block', + // Generated with script/tree-sitter-super-types tree-sitter-typescript/typescript class_body + abstract_class_declaration: 'class_body', + class: 'class_body', + class_declaration: 'class_body', + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + ['arrow_function', 'body'], + ['do_statement', 'body'], + ['else_clause', ''], + ['for_in_statement', 'body'], + ['for_statement', 'body'], + ['if_statement', 'consequence'], + ['while_statement', 'body'], + ['with_statement', 'body'], + ]), + /* startKeywords */[ + 'declare', + '=>', + 'try', + 'catch', + 'finally', + 'do', + 'for', + 'if', + 'else', + 'while', + 'with', + 'function', + 'function*', + 'class', + ], + /* blockNodeType */ 'statement_block', + /* emptyStatementType */ 'empty_statement', + /* curlyBraceLanguage */ true + ), + go: new RegexBasedBlockParser( + /* languageId */ 'go', + /* blockEmptyMatch */ '{}', + /* lineMatch */ /\b(func|if|else|for)\b/, + /* nodeMatch */ { + // Generated with script/tree-sitter-super-types tree-sitter-go block + communication_case: 'block', + default_case: 'block', + expression_case: 'block', + for_statement: 'block', + func_literal: 'block', + function_declaration: 'block', + if_statement: 'block', + labeled_statement: 'block', + method_declaration: 'block', + type_case: 'block', + }, + /* nodeTypesWithBlockOrStmtChild */ new Map() // Go always requires braces + ), + ruby: new RegexBasedBlockParser( + /* languageId */ 'ruby', + /* blockEmptyMatch */ 'end', + // Regex \b matches word boundaries - `->{}` has no word boundary. + /* lineMatch */ /\b(BEGIN|END|case|class|def|do|else|elsif|for|if|module|unless|until|while)\b|->/, + /* nodeMatch */ { + // Ruby works differently from other languages because there is no + // block-level node, instead we use the literal 'end' node to + // represent the end of a block. + begin_block: '}', + block: '}', + end_block: '}', + lambda: 'block', + for: 'do', + until: 'do', + while: 'do', + case: 'end', + do: 'end', + if: 'end', + method: 'end', + module: 'end', + unless: 'end', + do_block: 'end', + }, + // TODO(eaftan): Scour Ruby grammar for these + /* nodeTypesWithBlockOrStmtChild */ new Map() + ), + 'c-sharp': new TreeSitterBasedBlockParser( + /* languageId */ 'csharp', + /* nodeMatch */ { + // TODO -- unused in the current usage. + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + // TODO -- unused in the current usage. + ]), + /* startKeywords */[ + // TODO -- unused in the current usage. + ], + /* blockNodeType */ 'block', + /* emptyStatementType */ null, + /* curlyBraceLanguage */ true + ), + java: new TreeSitterBasedBlockParser( + /* languageId */ 'java', + /* nodeMatch */ { + // TODO -- unused in the current usage. + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + // TODO -- unused in the current usage. + ]), + /* startKeywords */[ + // TODO -- unused in the current usage. + ], + /* blockNodeType */ 'block', + /* emptyStatementType */ null, + /* curlyBraceLanguage */ true + ), + php: new TreeSitterBasedBlockParser( + /* languageId */ 'php', + /* nodeMatch */ { + // TODO -- unused in the current usage. + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + // TODO -- unused in the current usage. + ]), + /* startKeywords */[ + // TODO -- unused in the current usage. + ], + /* blockNodeType */ 'block', + /* emptyStatementType */ null, + /* curlyBraceLanguage */ true + ), + cpp: new TreeSitterBasedBlockParser( + /* languageId */ 'cpp', + /* nodeMatch */ { + // TODO -- unused in the current usage. + }, + /* nodeTypesWithBlockOrStmtChild */ new Map([ + // TODO -- unused in the current usage. + ]), + /* startKeywords */[ + // TODO -- unused in the current usage. + ], + /* blockNodeType */ 'block', + /* emptyStatementType */ null, + /* curlyBraceLanguage */ true + ), +}; + +export function getBlockParser(languageId: string): BlockParser { + if (!isSupportedLanguageId(languageId)) { + throw new Error(`Language ${languageId} is not supported`); + } + return wasmLanguageToBlockParser[languageIdToWasmLanguage(languageId)]; +} + +export async function isEmptyBlockStart(languageId: string, text: string, offset: number) { + if (!isSupportedLanguageId(languageId)) { + return false; + } + return getBlockParser(languageId).isEmptyBlockStart(text, offset); +} + +export async function isBlockBodyFinished(languageId: string, prefix: string, completion: string, offset: number) { + if (!isSupportedLanguageId(languageId)) { + return undefined; + } + return getBlockParser(languageId).isBlockBodyFinished(prefix, completion, offset); +} + +export async function getNodeStart(languageId: string, text: string, offset: number) { + if (!isSupportedLanguageId(languageId)) { + return; + } + return getBlockParser(languageId).getNodeStart(text, offset); +} diff --git a/completions-sample-code/vscode-node/prompt/src/prompt.ts b/completions-sample-code/vscode-node/prompt/src/prompt.ts new file mode 100644 index 0000000..714baa1 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/prompt.ts @@ -0,0 +1,99 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { SimilarFilesOptions } from './snippetInclusion/similarFiles'; + +/** + * Default PromptOptions are defined as constants to ensure the same values are shared + * between: + * - the class constructor + * - the EXP default flags + */ + +/** The maximum number of tokens in a completion. */ +export const DEFAULT_MAX_COMPLETION_LENGTH = 500; + +/** The maximum number of tokens in a prompt. */ +export const DEFAULT_MAX_PROMPT_LENGTH = 8192 - DEFAULT_MAX_COMPLETION_LENGTH; + +/** The maximal number of the final snippets to return. */ +export const DEFAULT_NUM_SNIPPETS = 4; + +/** + * The default threshold for choosing a cached suffix. + */ +export const DEFAULT_SUFFIX_MATCH_THRESHOLD = 10; + +/* The default allocation of the prompt to different components */ +export const DEFAULT_PROMPT_ALLOCATION_PERCENT = { + prefix: 35, + suffix: 15, + stableContext: 35, + volatileContext: 15, +} as const; + +export type PromptComponentId = keyof typeof DEFAULT_PROMPT_ALLOCATION_PERCENT; +export type PromptComponentAllocation = Record<PromptComponentId, number>; + +/** + * Information about a document, not including the offset. + */ +export interface DocumentInfo { + /** The file path of the document relative to its containing project or folder, if known. */ + relativePath?: string; + /** The URI of the document. We can't pass URI class instances directly due to limitations of passing objects to the worker thread. */ + uri: string; + /** The source text of the document. */ + source: string; + /** The language identifier of the document. */ + languageId: string; +} + +/** + * Information about a document, including an offset corresponding to + * the cursor position. + */ +export interface DocumentInfoWithOffset extends DocumentInfo { + /** The offset in the document where we want the completion (0-indexed, between characters). */ + offset: number; +} + +/** + * Information about a similar file. + */ +export type SimilarFileInfo = Omit<DocumentInfo, 'languageId'>; + +export type PromptOptions = { + /** The maximum prompt length in tokens */ + maxPromptLength: number; + /** The number of snippets to include */ + numberOfSnippets: number; + /** The percent of `maxPromptLength` to reserve for the suffix */ + suffixPercent: number; + /** The threshold (in percent) for declaring match of new suffix with existing suffix */ + suffixMatchThreshold: number; + /** The default parameters for the similar-files provider, for any language. */ + similarFilesOptions: SimilarFilesOptions; +}; + +/** + * A map that normalises common aliases of languageIds. + */ +const languageNormalizationMap: { [language: string]: string } = { + javascriptreact: 'javascript', + jsx: 'javascript', + typescriptreact: 'typescript', + jade: 'pug', + cshtml: 'razor', + c: 'cpp', +}; + +/** + * Return a normalized form of a language id, by lower casing and combining + * certain languageId's that are not considered distinct by promptlib. + */ +export function normalizeLanguageId(languageId: string): string { + languageId = languageId.toLowerCase(); + return languageNormalizationMap[languageId] ?? languageId; +} diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/cursorContext.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/cursorContext.ts new file mode 100644 index 0000000..1a3f993 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/cursorContext.ts @@ -0,0 +1,94 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Cursor contexts used by snippet providers, e.g. similar files. + * + * A 'cursor context' is quite similar to a prompt, but it is meant as a more + * basic, lightweight and ultimately myopic look at what the user is currently doing. + */ + +import { DocumentInfoWithOffset } from '../prompt'; +import { getTokenizer, TokenizerName } from '../tokenization'; + +/** + * Options for cursor context generation. + */ +type CursorContextOptions = { + /** The maximum cursor context length in tokens */ + maxTokenLength?: number; + + /** The maximum number of lines in a cursor context */ + maxLineCount?: number; + + /** TokenizerName for the tokenization */ + tokenizerName: TokenizerName; +}; + +const defaultCursorContextOptions: CursorContextOptions = { + tokenizerName: TokenizerName.o200k, +}; + +function cursorContextOptions(options?: Partial<CursorContextOptions>): CursorContextOptions { + return { ...defaultCursorContextOptions, ...options }; +} + +export interface CursorContextInfo { + /** The compiled context as a string */ + context: string; + /** The number of tokens in the context */ + tokenLength: number; + /** The number of lines in the context */ + lineCount: number; + /** TokenizerName for the tokenization */ + tokenizerName: TokenizerName; +} + +/** + * Return a cursor context corresponding to this document info. + * This is essentially a trimmed-down version of a prompt. + * + * If maxLineCount or maxTokenLength are 0, an empty context is returned + * If exactly one of `maxLineCount` or `maxTokenLength` is defined, the limit is applied for that one only + * If both are defined, we apply both conditions so end up using the shorter of the two constraints + * If both are undefined, the entire document up to the cursor is returned + */ +export function getCursorContext( + doc: DocumentInfoWithOffset, + options: Partial<CursorContextOptions> = {} +): CursorContextInfo { + const completeOptions = cursorContextOptions(options); + const tokenizer = getTokenizer(completeOptions.tokenizerName); + + if (completeOptions.maxLineCount !== undefined && completeOptions.maxLineCount < 0) { + throw new Error('maxLineCount must be non-negative if defined'); + } + if (completeOptions.maxTokenLength !== undefined && completeOptions.maxTokenLength < 0) { + throw new Error('maxTokenLength must be non-negative if defined'); + } + + if (completeOptions.maxLineCount === 0 || completeOptions.maxTokenLength === 0) { + return { + context: '', + lineCount: 0, + tokenLength: 0, + tokenizerName: completeOptions.tokenizerName, + }; + } + + let context = doc.source.slice(0, doc.offset); // Trim to cursor location, offset is a character location + if (completeOptions.maxLineCount !== undefined) { + context = context.split('\n').slice(-completeOptions.maxLineCount).join('\n'); + } + if (completeOptions.maxTokenLength !== undefined) { + context = tokenizer.takeLastLinesTokens(context, completeOptions.maxTokenLength); + } + return { + context, + lineCount: context.split('\n').length, + tokenLength: tokenizer.tokenLength(context), + tokenizerName: completeOptions.tokenizerName, + }; +} diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/jaccardMatching.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/jaccardMatching.ts new file mode 100644 index 0000000..c2a1472 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/jaccardMatching.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DocumentInfoWithOffset } from '../prompt'; +import { CursorContextInfo, getCursorContext } from './cursorContext'; +import { WindowedMatcher } from './selectRelevance'; +import { getBasicWindowDelineations } from './windowDelineations'; + +export class FixedWindowSizeJaccardMatcher extends WindowedMatcher { + private windowLength: number; + + private constructor(referenceDoc: DocumentInfoWithOffset, windowLength: number) { + super(referenceDoc); + this.windowLength = windowLength; + } + + static FACTORY = (windowLength: number) => { + return { + to: (referenceDoc: DocumentInfoWithOffset) => new FixedWindowSizeJaccardMatcher(referenceDoc, windowLength), + }; + }; + + protected id(): string { + return 'fixed:' + this.windowLength; + } + + protected getWindowsDelineations(lines: string[]): [number, number][] { + return getBasicWindowDelineations(this.windowLength, lines); + } + + protected _getCursorContextInfo(referenceDoc: DocumentInfoWithOffset): CursorContextInfo { + return getCursorContext(referenceDoc, { + maxLineCount: this.windowLength, + }); + } + + protected similarityScore(a: Set<string>, b: Set<string>): number { + return computeScore(a, b); + } +} + +/** + * Compute the Jaccard metric of number of elements in the intersection + * divided by number of elements in the union + */ +export function computeScore(a: Set<string>, b: Set<string>) { + const intersection = new Set(); + a.forEach(x => { + if (b.has(x)) { + intersection.add(x); + } + }); + return intersection.size / (a.size + b.size - intersection.size); +} diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/selectRelevance.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/selectRelevance.ts new file mode 100644 index 0000000..5343d4b --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/selectRelevance.ts @@ -0,0 +1,397 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DocumentInfo, DocumentInfoWithOffset, SimilarFileInfo } from '../prompt'; +import { CursorContextInfo } from './cursorContext'; +import { SnippetProviderType, SnippetSemantics, SnippetWithProviderInfo } from './snippets'; + +class FifoCache<T> { + private keys: string[] = []; + private cache: { [key: string]: T } = {}; + private size: number; + constructor(size: number) { + this.size = size; + } + put(key: string, value: T) { + this.cache[key] = value; + if (this.keys.length > this.size) { + this.keys.push(key); + const leavingKey = this.keys.shift() ?? ''; + delete this.cache[leavingKey]; + } + } + get(key: string): T | undefined { + return this.cache[key]; + } +} + +export interface ScoredSnippetMarker { + score: number; + startLine: number; + endLine: number; +} + +/** + * A snippet of code together with a relevance score. + * + * The scoring system assumes that a snippet with a **bigger** score is **more** relevant. + */ +export interface ScoredSnippet extends ScoredSnippetMarker { + snippet: string; + relativePath?: string; +} + +export enum SortOptions { + Ascending = 'ascending', + Descending = 'descending', + None = 'none', +} + +class Tokenizer { + private readonly stopsForLanguage: Set<string>; + constructor(doc: DocumentInfo) { + this.stopsForLanguage = SPECIFIC_STOPS.get(doc.languageId) ?? GENERIC_STOPS; + } + tokenize(a: string): Set<string> { + return new Set<string>(splitIntoWords(a).filter(x => !this.stopsForLanguage.has(x))); + } +} + +/** + * For a number of documents (the similar files), + * associate to each document and its kind of window computation (as key) + * the sequence b_1, ..., b_n, where + * b_i is the set of tokens in the ith window -- + * e.g. for window length 10, + * WINDOWED_TOKEN_SET_CACHE(doc)[0] + * holds the tokens in the first 10 lines of the document. + */ +const WINDOWED_TOKEN_SET_CACHE = new FifoCache<Set<string>[]>(20); + +/** + * For a given document, extracts the best matching snippets from other documents + * by comparing all of a set of windows in the object doc. + */ +export abstract class WindowedMatcher { + protected referenceDoc: DocumentInfoWithOffset; + protected tokenizer: Tokenizer; + + protected abstract id(): string; + protected abstract similarityScore(a: Set<string>, b: Set<string>): number; + /** + * Given an array of lines, returns an array of pairs <startLine, endLine> of indices, + * such that each pair is a window of lines to consider adding. + * startLine is inclusive, endLine is exclusive. + * @param lines Lines of a source text, in order + */ + protected abstract getWindowsDelineations(lines: string[]): [number, number][]; + + /** + * Subclasses should implement this method to return the desired context info for tokenization + * from the reference document. Will only be called after constructor is finished. + * The tokenizer used in WindowedMatcher is a simple tokenizer for Jaccard similarity, NOT an + * OpenAI model tokenizer. + */ + protected abstract _getCursorContextInfo(referenceDoc: DocumentInfoWithOffset): CursorContextInfo; + + protected constructor(referenceDoc: DocumentInfoWithOffset) { + this.referenceDoc = referenceDoc; + this.tokenizer = new Tokenizer(referenceDoc); // Just uses language info from referenceDoc + } + + /** + * Lazy getter for referenceTokens since it relies on properties + * that are not initialized in the constructor of WindowedMatcher + * but in the constructor of its subclasses. + */ + protected referenceTokensCache: Set<string> | undefined; + get referenceTokens(): Promise<Set<string>> { + return Promise.resolve(this.createReferenceTokens()); + } + + private createReferenceTokens(): Set<string> { + return (this.referenceTokensCache ??= this.tokenizer.tokenize( + this._getCursorContextInfo(this.referenceDoc).context + )); + } + + /** + * Returns a sorted array of snippets with their scores according to the sort option. + * @param snippets ScoredSnippet[] + * + */ + sortScoredSnippets(snippets: ScoredSnippetMarker[], sortOption = SortOptions.Descending): ScoredSnippetMarker[] { + return sortOption === SortOptions.Ascending + ? snippets.sort((snippetA, snippetB) => (snippetA.score > snippetB.score ? 1 : -1)) + : sortOption === SortOptions.Descending + ? snippets.sort((snippetA, snippetB) => (snippetA.score > snippetB.score ? -1 : 1)) + : snippets; + } + /** + * Returns all snippet markers with their scores. + * @param objectDoc + * + */ + async retrieveAllSnippets( + objectDoc: SimilarFileInfo, + sortOption = SortOptions.Descending + ): Promise<ScoredSnippetMarker[]> { + const snippets: ScoredSnippetMarker[] = []; + + if (objectDoc.source.length === 0 || (await this.referenceTokens).size === 0) { + return snippets; + } + + const lines = objectDoc.source.split('\n'); + const key = this.id() + ':' + objectDoc.source; + const tokensInWindows = WINDOWED_TOKEN_SET_CACHE.get(key) ?? []; + // if the tokens are not cached, we need to compute them + const needToComputeTokens = tokensInWindows.length === 0; + const tokenizedLines = needToComputeTokens ? lines.map(l => this.tokenizer.tokenize(l), this.tokenizer) : []; + + // Compute the windows with the score + for (const [index, [startLine, endLine]] of this.getWindowsDelineations(lines).entries()) { + if (needToComputeTokens) { + const tokensInWindow = new Set<string>(); + tokenizedLines + .slice(startLine, endLine) + .forEach(x => x.forEach(s => tokensInWindow.add(s), tokensInWindow)); + tokensInWindows.push(tokensInWindow); + } + // Now tokensInWindows[index] contains the tokens in the window, whether we just computed them or not + const tokensInWindow = tokensInWindows[index]; + const score = this.similarityScore(tokensInWindow, await this.referenceTokens); + + // If snippets overlap, keep the one with highest score. + // Note: Assuming the getWindowsDelineations function returns windows in sorted ascending line ranges. + if (snippets.length && startLine > 0 && snippets[snippets.length - 1].endLine > startLine) { + if (snippets[snippets.length - 1].score < score) { + snippets[snippets.length - 1].score = score; + snippets[snippets.length - 1].startLine = startLine; + snippets[snippets.length - 1].endLine = endLine; + } + continue; + } + + snippets.push({ + score, + startLine, + endLine, + }); + } + + // If we didn't get the token sets from the cache, time to put them there! + if (needToComputeTokens) { + WINDOWED_TOKEN_SET_CACHE.put(key, tokensInWindows); + } + + return this.sortScoredSnippets(snippets, sortOption); + } + + findMatches(objectDoc: SimilarFileInfo, maxSnippetsPerFile: number): Promise<SnippetWithProviderInfo[]> { + const snippet = this.findBestMatch(objectDoc, maxSnippetsPerFile); + return snippet; + } + + /** + * Returns the snippet from the object document + * that is most similar to the reference Document + * together with its Jaccard score + * + * @param objectDoc + */ + async findBestMatch(objectDoc: SimilarFileInfo, maxSnippetsPerFile: number): Promise<SnippetWithProviderInfo[]> { + if (objectDoc.source.length === 0 || (await this.referenceTokens).size === 0) { + return []; + } + const lines = objectDoc.source.split('\n'); + const snippets = await this.retrieveAllSnippets(objectDoc, SortOptions.Descending); + + // safe guard against empty lists + if (snippets.length === 0) { + return []; + } + + const bestSnippets: SnippetWithProviderInfo[] = []; + + for (let i = 0; i < snippets.length && i < maxSnippetsPerFile; i++) { + // Skip null scored snippets. + if (snippets[i].score !== 0) { + // Get the snippet's text. + const snippetCode = lines.slice(snippets[i].startLine, snippets[i].endLine).join('\n'); + bestSnippets.push({ + snippet: snippetCode, + semantics: SnippetSemantics.Snippet, + provider: SnippetProviderType.SimilarFiles, + ...snippets[i], + }); + } + } + + return bestSnippets; + } +} + +/** + * Split by non-alphanumeric characters + */ +export function splitIntoWords(a: string): string[] { + return a.split(/[^a-zA-Z0-9]/).filter(x => x.length > 0); +} + +const ENGLISH_STOPS = new Set([ + // - pronouns + 'we', + 'our', + 'you', + 'it', + 'its', + 'they', + 'them', + 'their', + 'this', + 'that', + 'these', + 'those', + // - verbs + 'is', + 'are', + 'was', + 'were', + 'be', + 'been', + 'being', + 'have', + 'has', + 'had', + 'having', + 'do', + 'does', + 'did', + 'doing', + 'can', + 'don', + 't', + 's', + 'will', + 'would', + 'should', + // - wh-words + 'what', + 'which', + 'who', + 'when', + 'where', + 'why', + 'how', + // - articles + 'a', + 'an', + 'the', + // - prepositions + 'and', + 'or', + 'not', + 'no', + 'but', + 'because', + 'as', + 'until', + 'again', + 'further', + 'then', + 'once', + 'here', + 'there', + 'all', + 'any', + 'both', + 'each', + 'few', + 'more', + 'most', + 'other', + 'some', + 'such', + 'above', + 'below', + 'to', + 'during', + 'before', + 'after', + 'of', + 'at', + 'by', + 'about', + 'between', + 'into', + 'through', + 'from', + 'up', + 'down', + 'in', + 'out', + 'on', + 'off', + 'over', + 'under', + 'only', + 'own', + 'same', + 'so', + 'than', + 'too', + 'very', + 'just', + 'now', +]); + +/** + * A generic set of stops for any programming language + */ +const GENERIC_STOPS = new Set([ + // words that are common in programming languages + 'if', + 'then', + 'else', + 'for', + 'while', + 'with', + 'def', + 'function', + 'return', + 'TODO', + 'import', + 'try', + 'catch', + 'raise', + 'finally', + 'repeat', + 'switch', + 'case', + 'match', + 'assert', + 'continue', + 'break', + 'const', + 'class', + 'enum', + 'struct', + 'static', + 'new', + 'super', + 'this', + 'var', + // words that are common in English comments: + ...ENGLISH_STOPS, +]); + +/** + * Specific stops for certain languages + * Note that ENGLISH_STOPS need to be added to this set if they are to be included + */ +const SPECIFIC_STOPS: Map<string, Set<string>> = new Map([ + // none yet +]); diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/similarFiles.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/similarFiles.ts new file mode 100644 index 0000000..22dd0ee --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/similarFiles.ts @@ -0,0 +1,119 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DocumentInfoWithOffset, SimilarFileInfo } from '../prompt'; +import { FixedWindowSizeJaccardMatcher } from './jaccardMatching'; +import { SnippetWithProviderInfo } from './snippets'; +import { BlockTokenSubsetMatcher } from './subsetMatching'; + +const DEFAULT_SNIPPET_THRESHOLD = 0.0; +const DEFAULT_SNIPPET_WINDOW_SIZE = 60; +const DEFAULT_MAX_TOP_SNIPPETS = 4; +const DEFAULT_MAX_SNIPPETS_PER_FILE = 1; +const DEFAULT_MAX_NUMBER_OF_FILES = 20; +const DEFAULT_MAX_CHARACTERS_PER_FILE = 10000; + +export interface SimilarFilesOptions { + snippetLength: number; + threshold: number; + maxTopSnippets: number; + maxCharPerFile: number; + maxNumberOfFiles: number; + maxSnippetsPerFile: number; + useSubsetMatching?: boolean; +} + +export const defaultSimilarFilesOptions: SimilarFilesOptions = { + snippetLength: DEFAULT_SNIPPET_WINDOW_SIZE, + threshold: DEFAULT_SNIPPET_THRESHOLD, + maxTopSnippets: DEFAULT_MAX_TOP_SNIPPETS, + maxCharPerFile: DEFAULT_MAX_CHARACTERS_PER_FILE, + maxNumberOfFiles: DEFAULT_MAX_NUMBER_OF_FILES, + maxSnippetsPerFile: DEFAULT_MAX_SNIPPETS_PER_FILE, + useSubsetMatching: false, +}; + +export const conservativeFilesOptions: SimilarFilesOptions = { + snippetLength: 10, + threshold: 0.3, + maxTopSnippets: 1, + maxCharPerFile: DEFAULT_MAX_CHARACTERS_PER_FILE, + maxNumberOfFiles: DEFAULT_MAX_NUMBER_OF_FILES, + maxSnippetsPerFile: 1, +}; + +export const nullSimilarFilesOptions: SimilarFilesOptions = { + snippetLength: 0, + threshold: 1, + maxTopSnippets: 0, + maxCharPerFile: 0, + maxNumberOfFiles: 0, + maxSnippetsPerFile: 0, +}; + +// Default similarity parameters for languageId === 'cpp'. +export const defaultCppSimilarFilesOptions: SimilarFilesOptions = { + snippetLength: 60, + threshold: 0.0, + maxTopSnippets: 16, + maxCharPerFile: 100000, + maxNumberOfFiles: 200, + maxSnippetsPerFile: 4, +}; + +function getMatcher(doc: DocumentInfoWithOffset, selection: SimilarFilesOptions) { + const matcherFactory = selection.useSubsetMatching + ? BlockTokenSubsetMatcher.FACTORY(selection.snippetLength) + : FixedWindowSizeJaccardMatcher.FACTORY(selection.snippetLength); + return matcherFactory.to(doc); +} + +/** + * @returns A SnippetWithProviderInfo describing the best matches from similar files. + */ +export async function getSimilarSnippets( + doc: DocumentInfoWithOffset, + similarFiles: SimilarFileInfo[], + options: SimilarFilesOptions +): Promise<SnippetWithProviderInfo[]> { + const matcher = getMatcher(doc, options); + if (options.maxTopSnippets === 0) { + return []; + } + + const snippets = ( + await similarFiles + // filter out absurdly long or absurdly many open files + .filter(similarFile => similarFile.source.length < options.maxCharPerFile && similarFile.source.length > 0) + // slice(0) duplicates an array + .slice(0, options.maxNumberOfFiles) + .reduce( + async ( + acc, + similarFile // accumulator of all snippets from all similarFiles + ) => + (await acc).concat( + (await matcher.findMatches(similarFile, options.maxSnippetsPerFile)).map(snippet => ({ + relativePath: similarFile.relativePath, + ...snippet, + })) + ), + Promise.resolve([] as SnippetWithProviderInfo[]) + ) + ) + .filter( + similarFile => + // remove files that had no match at all + similarFile.score && + similarFile.snippet && + // remove files that had a low score + similarFile.score > options.threshold + ) + // order them with best (highest scores) last + .sort((a, b) => a.score - b.score) + // take the best options from the end + .slice(-options.maxTopSnippets); + return snippets; +} diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/snippets.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/snippets.ts new file mode 100644 index 0000000..37c7cdd --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/snippets.ts @@ -0,0 +1,76 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ScoredSnippet } from './selectRelevance'; + +/** Indicates what provider produced a given snippet. */ +export enum SnippetProviderType { + SimilarFiles = 'similar-files', + Path = 'path', +} + +/** + * The semantics of a snippet. For example, some providers + * might always produce a snippet that is a complete function + * whereas others might produce a snippet that are inherhently + * partial. + */ +export enum SnippetSemantics { + /** The contents of the snippet is a function. */ + Function = 'function', + /** The contents of the snippet is an unspecified snippet. */ + Snippet = 'snippet', + /** Contains multiple snippets of type snippet */ + Snippets = 'snippets', + /** The following are from hover text */ + Variable = 'variable', + Parameter = 'parameter', + Method = 'method', + Class = 'class', + Module = 'module', + Alias = 'alias', + Enum = 'enum member', + Interface = 'interface', +} + +/** Extends a ScoredSnippet with information about its provider. */ +export interface SnippetWithProviderInfo extends ScoredSnippet { + /** The provider that created this snippet. */ + provider: SnippetProviderType; + /** The semantical meaning of the snippet's contents. */ + semantics: SnippetSemantics; +} + +type SnippetToAnnounce = Pick<SnippetWithProviderInfo, 'snippet' | 'semantics' | 'relativePath'>; + +/** + * A map from semantics enum to a human / LLM-readable label that we + * include when announcing a snippet. + */ +const snippetSemanticsToString: { [key in SnippetSemantics]: string } = { + [SnippetSemantics.Function]: 'function', + [SnippetSemantics.Snippet]: 'snippet', + [SnippetSemantics.Snippets]: 'snippets', + [SnippetSemantics.Variable]: 'variable', + [SnippetSemantics.Parameter]: 'parameter', + [SnippetSemantics.Method]: 'method', + [SnippetSemantics.Class]: 'class', + [SnippetSemantics.Module]: 'module', + [SnippetSemantics.Alias]: 'alias', + [SnippetSemantics.Enum]: 'enum member', + [SnippetSemantics.Interface]: 'interface', +}; + +/** + * Formats a snippet for inclusion in the prompt. + */ +export function announceSnippet(snippet: SnippetToAnnounce) { + const semantics = snippetSemanticsToString[snippet.semantics]; + const pluralizedSemantics = [SnippetSemantics.Snippets].includes(snippet.semantics) ? 'these' : 'this'; + const headline = snippet.relativePath + ? `Compare ${pluralizedSemantics} ${semantics} from ${snippet.relativePath}:` + : `Compare ${pluralizedSemantics} ${semantics}:`; + return { headline, snippet: snippet.snippet }; +} diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/subsetMatching.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/subsetMatching.ts new file mode 100644 index 0000000..e2eef47 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/subsetMatching.ts @@ -0,0 +1,159 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { parseTreeSitter } from '../parse'; +import { DocumentInfoWithOffset } from '../prompt'; +import { CursorContextInfo, getCursorContext } from './cursorContext'; +import { WindowedMatcher } from './selectRelevance'; +import { getBasicWindowDelineations } from './windowDelineations'; +import Parser from 'web-tree-sitter'; + +/** + * Implements an evolution of the FixedWindowSizeJaccardMatcher that is different in two ways. + * 1. The source tokens window is the enclosing class member, as determined by Tree-Sitter. + * 2. The scoring algorithm is a unidirectional set membership check (count of items from A that exist in B) + * rather than a set difference. + */ +export class BlockTokenSubsetMatcher extends WindowedMatcher { + private windowLength: number; + + private constructor(referenceDoc: DocumentInfoWithOffset, windowLength: number) { + super(referenceDoc); + this.windowLength = windowLength; + } + + static FACTORY = (windowLength: number) => { + return { + to: (referenceDoc: DocumentInfoWithOffset) => new BlockTokenSubsetMatcher(referenceDoc, windowLength), + }; + }; + + protected id(): string { + return 'fixed:' + this.windowLength; + } + + protected getWindowsDelineations(lines: string[]): [number, number][] { + return getBasicWindowDelineations(this.windowLength, lines); + } + + protected _getCursorContextInfo(referenceDoc: DocumentInfoWithOffset): CursorContextInfo { + return getCursorContext(referenceDoc, { + maxLineCount: this.windowLength, + }); + } + + override get referenceTokens(): Promise<Set<string>> { + return this.createReferenceTokensForLanguage(); + } + + private async createReferenceTokensForLanguage(): Promise<Set<string>> { + if (this.referenceTokensCache) { + return this.referenceTokensCache; + } + + // Syntax aware reference tokens uses tree-sitter based parsing to identify the bounds of the current + // method and extracts tokens from just that span for use as the reference set. + this.referenceTokensCache = BlockTokenSubsetMatcher.syntaxAwareSupportsLanguage(this.referenceDoc.languageId) + ? await this.syntaxAwareReferenceTokens() + : await super.referenceTokens; + + return this.referenceTokensCache; + } + + private async syntaxAwareReferenceTokens(): Promise<Set<string>> { + // See if there is an enclosing class or type member. + const start = (await this.getEnclosingMemberStart(this.referenceDoc.source, this.referenceDoc.offset)) + ?.startIndex; + const end = this.referenceDoc.offset; + + // If not, fallback to the 60-line chunk behavior. + const text = start + ? this.referenceDoc.source.slice(start, end) + : getCursorContext(this.referenceDoc, { + maxLineCount: this.windowLength, + }).context; + + // Extract the tokens. + return this.tokenizer.tokenize(text); + } + + private static syntaxAwareSupportsLanguage(languageId: string): boolean { + switch (languageId) { + case 'csharp': + return true; + default: + return false; + } + } + + protected similarityScore(a: Set<string>, b: Set<string>): number { + return computeScore(a, b); + } + + async getEnclosingMemberStart(text: string, offset: number): Promise<Parser.SyntaxNode | undefined> { + let tree: Parser.Tree | undefined; + + try { + tree = await parseTreeSitter(this.referenceDoc.languageId, text); + + let nodeAtPos: Parser.SyntaxNode | undefined = tree.rootNode.namedDescendantForIndex(offset); + + while (nodeAtPos) { + // For now, hard code for C#. + if (BlockTokenSubsetMatcher.isMember(nodeAtPos) || BlockTokenSubsetMatcher.isBlock(nodeAtPos)) { + break; + } + + nodeAtPos = nodeAtPos.parent ?? undefined; + } + + return nodeAtPos; + } finally { + tree?.delete(); + } + } + + static isMember(node: Parser.SyntaxNode | undefined): boolean { + // For now, hard code for C#. + switch (node?.type) { + case 'method_declaration': + case 'property_declaration': + case 'field_declaration': + case 'constructor_declaration': + return true; + default: + return false; + } + } + + static isBlock(node: Parser.SyntaxNode | undefined): boolean { + // For now, hard code for C#. + switch (node?.type) { + case 'class_declaration': + case 'struct_declaration': + case 'record_declaration': + case 'enum_declaration': + case 'interface_declaration': + return true; + default: + return false; + } + } +} + +/** + * Count the number of unique tokens from B that are also in A. + */ +function computeScore(a: Set<string>, b: Set<string>) { + const subsetOverlap = new Set(); + + b.forEach(x => { + if (a.has(x)) { + subsetOverlap.add(x); + } + }); + + return subsetOverlap.size; +} diff --git a/completions-sample-code/vscode-node/prompt/src/snippetInclusion/windowDelineations.ts b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/windowDelineations.ts new file mode 100644 index 0000000..64eae5a --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/snippetInclusion/windowDelineations.ts @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IndentationTree } from '../indentation/classes'; +import { clearLabels, visitTree } from '../indentation/manipulation'; +import { parseTree } from '../indentation/parsing'; + +/** + * Returns a list of (startline, endline) pairs representing fixed size windows + * + * @param windowLength length of fixed size window + * @param lines lines to extract fixed size windows from + * @returns list of (startline, endline) pairs + */ +export function getBasicWindowDelineations(windowLength: number, lines: string[]): [number, number][] { + const windows: [number, number][] = []; + const length = lines.length; + if (length === 0) { + return []; + } + if (length < windowLength) { + // if not long enough to reach a single window length, return full document + return [[0, length]]; + } + for (let startLine = 0; startLine < length - windowLength + 1; startLine++) { + windows.push([startLine, startLine + windowLength]); + } + return windows; +} + +/** + * Calculate all windows like with the following properties: + * - they are all of length <= maxLength + * - they are all of length >= minLength + * - except if they are followed by enough blank lines to reach length >= minLength + * - they are a contiguous subsequence from [parentline, child1, child2, ..., childn] + * - which neither starts nor ends with a blank line + * Note that windows of the form "parent with all its children" could + * appear in different ways with that definition, + * e.g. as "childi" of its parent, and as "parent, child1, ..., childn" where the parent is itself. + * Nevertheless, it will only be listed once. + * @param lines + */ +export function getIndentationWindowsDelineations( + lines: string[], + languageId: string, + minLength: number, + maxLength: number +): [number, number][] { + // Deal with degenerate cases + if (lines.length < minLength || maxLength === 0) { + return []; + } + + const windows: [number, number][] = []; + // For each node, keep track of how long its children extend, or whether it can't be included in a window anyhow + type TreeLabel = { totalLength: number; firstLineAfter: number }; + // Todo: add groupBlocks here as well + const labeledTree = clearLabels(parseTree(lines.join('\n'), languageId)) as IndentationTree<TreeLabel>; + visitTree( + labeledTree, + node => { + if (node.type === 'blank') { + node.label = { totalLength: 1, firstLineAfter: node.lineNumber + 1 }; + return; + } + // Statistics to gather on the way, to be consumed by parents + let totalLength = node.type === 'line' ? 1 : 0; + let firstLineAfter = node.type === 'line' ? node.lineNumber + 1 : NaN; + // we consider intervals [a, b] which correspond to including children number a (-1 means parent) through b exclusive. + // the window start and end lines are computed here, such that startLine (inclusive) to endLine (exclusive) covers the window + function getStartLine(a: number) { + return a === -1 + ? firstLineAfter - totalLength + : node.subs[a].label!.firstLineAfter - node.subs[a].label!.totalLength; + } + function getEndLine(b: number, startLine: number) { + return b === 0 ? startLine + 1 : node.subs[b - 1].label!.firstLineAfter; + } + // iteratively go through candidates for [a, b[: + // if from a to including b would be too long, add the window a to b exclusive and increase a as far as necessary, otherwise increase b + // a = -1 will mean: include the parent + let a = node.type === 'line' ? -1 : 0; // if the parent is a line, consider using it + let lengthFromAToBInclusive = node.type === 'line' ? 1 : 0; // if so, the length is 1, otherwise 0 + let lastBThatWasntABlank = 0; + for (let b = 0; b < node.subs.length; b++) { + // don't let the window start with blank lines + while (a >= 0 && a < node.subs.length && node.subs[a].type === 'blank') { + lengthFromAToBInclusive -= node.subs[a].label!.totalLength; + a++; + } + if (node.subs[b].type !== 'blank') { + lastBThatWasntABlank = b; + } + // add subs[b] to the window + firstLineAfter = node.subs[b].label!.firstLineAfter; + totalLength += node.subs[b].label!.totalLength; + lengthFromAToBInclusive += node.subs[b].label!.totalLength; + if (lengthFromAToBInclusive > maxLength) { + const startLine = getStartLine(a); + const endLine = getEndLine(b, startLine); + const endLineTrimmedForBlanks = + lastBThatWasntABlank === b ? endLine : getEndLine(lastBThatWasntABlank, startLine); + // for the test, note that blanks count for getting us over the minLength: + if (minLength <= endLine - startLine) { + windows.push([startLine, endLineTrimmedForBlanks]); + } + while (lengthFromAToBInclusive > maxLength) { + // remove subs[a] from the window + lengthFromAToBInclusive -= + a === -1 + ? node.type === 'line' + ? 1 + : // this cannot happen: if not a line, we start with a = 0 unless it's a line + 0 + : node.subs[a].label!.totalLength; + a++; + } + } + } + // if there's anything left to add (a < b), do it + if (a < node.subs.length) { + const startLine = getStartLine(a); + const endLine = firstLineAfter; + const endLineTrimmedForBlanks = + a === -1 ? endLine : node.subs[lastBThatWasntABlank].label!.firstLineAfter; + // note: even if fillUpWindowWithPartOfNextNeighbor is true, + // there is no next similar file here, so nothing to extend the window to + if (minLength <= endLine - startLine) { + windows.push([startLine, endLineTrimmedForBlanks]); + } + // Set the node's label + } + node.label = { totalLength, firstLineAfter }; + }, + 'bottomUp' + ); + // windows is an array of [start, end] pairs, + // but some may appear twice, and should be removed + return windows + .sort((a, b) => a[0] - b[0] || a[1] - b[1]) + .filter((a, i, arr) => i === 0 || a[0] !== arr[i - 1][0] || a[1] !== arr[i - 1][1]); +} diff --git a/completions-sample-code/vscode-node/prompt/src/suffixMatchCriteria.ts b/completions-sample-code/vscode-node/prompt/src/suffixMatchCriteria.ts new file mode 100644 index 0000000..b444a66 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/suffixMatchCriteria.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +interface ScoredSuffix { + score: number; +} + +export function findEditDistanceScore(a: number[], b: number[]): ScoredSuffix { + if (a.length === 0 || b.length === 0) { + return { score: a.length + b.length }; + } + + const matrix = Array.from({ length: a.length }).map(() => Array.from({ length: b.length }).map(() => 0)); + for (let i = 0; i < a.length; i++) { + matrix[i][0] = i; + } + + for (let i = 0; i < b.length; i++) { + matrix[0][i] = i; + } + + for (let j = 0; j < b.length; j++) { + for (let i = 0; i < a.length; i++) { + matrix[i][j] = Math.min( + (i === 0 ? j : matrix[i - 1][j]) + 1, + (j === 0 ? i : matrix[i][j - 1]) + 1, + (i === 0 || j === 0 ? Math.max(i, j) : matrix[i - 1][j - 1]) + (a[i] === b[j] ? 0 : 1) + ); + } + } + + return { score: matrix[a.length - 1][b.length - 1] }; +} diff --git a/completions-sample-code/vscode-node/prompt/src/test/components/hooks.test.ts b/completions-sample-code/vscode-node/prompt/src/test/components/hooks.test.ts new file mode 100644 index 0000000..76c4d94 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/components/hooks.test.ts @@ -0,0 +1,226 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { UseData, UseState } from '../../components/hooks'; +import * as assert from 'assert'; +import { isNumber, isString } from './testHelpers'; + +suite('Hooks', function () { + suite('Use State', function () { + test('stores state', function () { + const state = new UseState([]); + + const [value] = state.useState(0); + + assert.deepStrictEqual(value, 0); + }); + + test('accepts undefined as initial state', function () { + const state = new UseState([]); + + const [value] = state.useState(undefined); + + assert.deepStrictEqual(value, undefined); + }); + + test('accepts no value as initial state', function () { + const state = new UseState([]); + + const [value] = state.useState(); + + assert.deepStrictEqual(value, undefined); + }); + + test('marks state as changed when updating state', function () { + const state = new UseState([]); + + const [_, setValue] = state.useState(0); + setValue(1); + + assert.strictEqual(state.hasChanged(), true); + }); + + test('stores state across use state instances', function () { + const rawState: number[] = []; + + const [value, setValue] = new UseState(rawState).useState(0); + setValue(1); + const [newValue] = new UseState(rawState).useState(0); + + assert.deepStrictEqual(value, 0); + assert.deepStrictEqual(newValue, 1); + }); + + test('multiple use state invocations produce separate state', function () { + const rawState: number[] = []; + const state = new UseState(rawState); + + const [value1] = state.useState(0); + const [value2] = state.useState('test'); + + assert.deepStrictEqual(value1, 0); + assert.deepStrictEqual(value2, 'test'); + }); + + test('accepts function as initial state', function () { + const state = new UseState([]); + const initializer = () => 42; + + const [value] = state.useState(initializer); + + assert.deepStrictEqual(value, 42); + }); + + test('setState accepts function to update state', function () { + const rawState: number[] = []; + const state1 = new UseState(rawState); + const [value, setValue] = state1.useState(1); + const state2 = new UseState(rawState); + + setValue(prev => prev + 1); + const [updatedValue] = state2.useState(0); + + assert.deepStrictEqual(value, 1); + assert.deepStrictEqual(updatedValue, 2); + assert.strictEqual(state1.hasChanged(), true); + }); + + test('maintains separate states when multiple instances share raw state', function () { + const rawState: number[] = []; + const state1 = new UseState(rawState); + const state2 = new UseState(rawState); + + const [count1, setCount1] = state1.useState(0); + setCount1(5); + const [count2] = state2.useState(0); + + assert.strictEqual(count1, 0); + assert.strictEqual(count2, 5); + }); + + test('hasChanged returns false before any setState calls', function () { + const state = new UseState([]); + state.useState(0); + + assert.strictEqual(state.hasChanged(), false); + }); + }); + + suite('Use Data', function () { + test('stores data callback for type', async function () { + const useData = new UseData(() => { }); + let data = ''; + + useData.useData(isString, (value: string) => { + data = value; + }); + await useData.updateData('test'); + + assert.deepStrictEqual(data, 'test'); + }); + + test('stores async data callback for type', async function () { + const useData = new UseData(() => { }); + let data = ''; + + useData.useData(isString, async (value: string) => { + await Promise.resolve(); + data = value; + }); + await useData.updateData('test'); + + assert.deepStrictEqual(data, 'test'); + }); + + test('stores multiple data callbacks for type', async function () { + const useData = new UseData(() => { }); + let data1 = ''; + let data2 = ''; + + useData.useData(isString, (value: string) => { + data1 = value; + }); + useData.useData(isString, (value: string) => { + data2 = value; + }); + await useData.updateData('test'); + + assert.deepStrictEqual(data1, 'test'); + assert.deepStrictEqual(data2, 'test'); + }); + + test('stores multiple async data callbacks for type', async function () { + const useData = new UseData(() => { }); + let data1 = ''; + let data2 = ''; + + useData.useData(isString, async (value: string) => { + await Promise.resolve(); + data1 = value; + }); + useData.useData(isString, async (value: string) => { + await Promise.resolve(); + data2 = value; + }); + await useData.updateData('test'); + + assert.deepStrictEqual(data1, 'test'); + assert.deepStrictEqual(data2, 'test'); + }); + + test('stores multiple data callbacks for different types', async function () { + const useData = new UseData(() => { }); + let data1 = ''; + let data2 = 0; + + useData.useData(isString, (value: string) => { + data1 = value; + }); + useData.useData(isNumber, (value: number) => { + data2 = value; + }); + await useData.updateData('test'); + await useData.updateData(23); + + assert.deepStrictEqual(data1, 'test'); + assert.deepStrictEqual(data2, 23); + }); + + test('updates data for subscribed types only', async function () { + const useData = new UseData(() => { }); + let data = ''; + + useData.useData(isString, (value: string) => { + data = value; + }); + await useData.updateData(23); + + assert.deepStrictEqual(data, ''); + }); + + test('updates data measures time to update', async function () { + let time = 0; + const useData = new UseData(updateTimeMs => { + time = updateTimeMs; + }); + let data = ''; + + useData.useData(isString, (value: string) => { + data = value; + }); + await useData.updateData(23); + + assert.deepStrictEqual(data, ''); + assert.ok(time > 0); + }); + + test('updates data measures time only if data hooks are present', async function () { + const useData = new UseData(updateTimeMs => { + throw new Error('Should not be called'); + }); + await useData.updateData(23); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/components/jsx-runtime.test.ts.off b/completions-sample-code/vscode-node/prompt/src/test/components/jsx-runtime.test.ts.off new file mode 100644 index 0000000..64b3498 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/components/jsx-runtime.test.ts.off @@ -0,0 +1,33 @@ +import {Fragment, jsx} from '#jsx/jsx-runtime'; +import {ComponentContext} from '#prompt/components/components'; +import * as assert from 'assert'; + +suite('JSX/TSX', function () { + test('Produces element from functional component', function () { + const fn = (props: unknown, context: ComponentContext) => []; + const element = jsx(fn, {children: ['Hello world']}); + + assert.deepStrictEqual(element.type, fn); + assert.deepStrictEqual(element.props, { + children: ['Hello world'], + }); + }); + + test('Produces element from functional component with key', function () { + const fn = (props: unknown, context: ComponentContext) => []; + const element = jsx(fn, {children: ['Hello world']}, 'key'); + + assert.deepStrictEqual(element.type, fn); + assert.deepStrictEqual(element.props, { + children: ['Hello world'], + key: 'key', + }); + }); + + test('Produces fragment function', function () { + const element = Fragment(['Hello world']); + + assert.deepStrictEqual(element.type, 'f'); + assert.deepStrictEqual(element.children, ['Hello world']); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/components/reconciler.test.tsx b/completions-sample-code/vscode-node/prompt/src/test/components/reconciler.test.tsx new file mode 100644 index 0000000..d689359 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/components/reconciler.test.tsx @@ -0,0 +1,762 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../jsx-runtime */ +import { Chunk, ComponentContext, PromptElement, PromptElementProps, Text } from '../../components/components'; +import { Dispatch, StateUpdater } from '../../components/hooks'; +import { VirtualPromptReconciler } from '../../components/reconciler'; +import * as assert from 'assert'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { extractNodesWitPath, isNumber, isString } from './testHelpers'; + +suite('Virtual prompt reconciler', function () { + test('computes paths for virtual prompt nodes', function () { + const MyNestedComponent = () => { + return ( + <> + <Text>Hola</Text> + <Text>Adios</Text> + </> + ); + }; + + const prompt = ( + <> + <MyNestedComponent /> + <Text>Intermediate</Text> + <MyNestedComponent /> + </> + ); + + const reconciler = new VirtualPromptReconciler(prompt); + const result = reconciler.reconcile(); + + const orderedPaths = extractNodesWitPath(result!); + + // Assert expected paths + assert.deepStrictEqual(orderedPaths, [ + '$.f', + '$.f[0].MyNestedComponent', + '$.f[0].MyNestedComponent[0].f', + '$.f[0].MyNestedComponent[0].f[0].Text', + '$.f[0].MyNestedComponent[0].f[0].Text[0]', + '$.f[0].MyNestedComponent[0].f[1].Text', + '$.f[0].MyNestedComponent[0].f[1].Text[0]', + '$.f[1].Text', + '$.f[1].Text[0]', + '$.f[2].MyNestedComponent', + '$.f[2].MyNestedComponent[0].f', + '$.f[2].MyNestedComponent[0].f[0].Text', + '$.f[2].MyNestedComponent[0].f[0].Text[0]', + '$.f[2].MyNestedComponent[0].f[1].Text', + '$.f[2].MyNestedComponent[0].f[1].Text[0]', + ]); + + // Assert uniqueness of paths + assert.deepStrictEqual([...new Set(orderedPaths)], orderedPaths); + }); + + test('computes paths for virtual prompt nodes with keys', function () { + const MyNestedComponent = () => { + return ( + <> + <Text>Hola</Text> + <Text key={23}>Adios</Text> + </> + ); + }; + + const prompt = ( + <> + <MyNestedComponent /> + <Chunk> + <Text key={'key-1'}>Text with key</Text> + </Chunk> + <MyNestedComponent /> + </> + ); + + const reconciler = new VirtualPromptReconciler(prompt); + const result = reconciler.reconcile(); + + const orderedPaths = extractNodesWitPath(result!); + + assert.deepStrictEqual(orderedPaths, [ + '$.f', + '$.f[0].MyNestedComponent', + '$.f[0].MyNestedComponent[0].f', + '$.f[0].MyNestedComponent[0].f[0].Text', + '$.f[0].MyNestedComponent[0].f[0].Text[0]', + '$.f[0].MyNestedComponent[0].f["23"].Text', + '$.f[0].MyNestedComponent[0].f["23"].Text[0]', + '$.f[1].Chunk', + '$.f[1].Chunk["key-1"].Text', + '$.f[1].Chunk["key-1"].Text[0]', + '$.f[2].MyNestedComponent', + '$.f[2].MyNestedComponent[0].f', + '$.f[2].MyNestedComponent[0].f[0].Text', + '$.f[2].MyNestedComponent[0].f[0].Text[0]', + '$.f[2].MyNestedComponent[0].f["23"].Text', + '$.f[2].MyNestedComponent[0].f["23"].Text[0]', + ]); + + // Assert uniqueness of paths + assert.deepStrictEqual([...new Set(orderedPaths)], orderedPaths); + }); + + test('rejects duplicate keys on same level in initial prompt', function () { + const prompt = ( + <> + <Text key={23}>Hola</Text> + <Text key={23}>Adios</Text> + </> + ); + + try { + new VirtualPromptReconciler(prompt); + assert.fail('Should have thrown an error'); + } catch (e) { + assert.equal((e as Error).message, 'Duplicate keys found: 23'); + } + }); + + test('rejects multiple duplicate keys on same level in initial prompt', function () { + const prompt = ( + <> + <Text key={23}>Hola</Text> + <Text key={23}>Adios</Text> + <Text key={'aKey'}>Hola</Text> + <Text key={'aKey'}>Adios</Text> + </> + ); + + try { + new VirtualPromptReconciler(prompt); + assert.fail('Should have thrown an error'); + } catch (e) { + assert.equal((e as Error).message, 'Duplicate keys found: 23, aKey'); + } + }); + + test('rejects duplicate keys on same level during reconciliation', function () { + let outerSetCount: Dispatch<StateUpdater<number>>; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [count, setCount] = context.useState(1); + + outerSetCount = setCount; + + return ( + <> + {Array.from({ length: count }).map((_, i) => ( + <Text key={23}>Text {i}</Text> + ))} + </> + ); + }; + + const reconciler = new VirtualPromptReconciler(<MyTestComponent />); + + outerSetCount!(2); + + try { + reconciler.reconcile(); + assert.fail('Should have thrown an error'); + } catch (e) { + assert.equal((e as Error).message, 'Duplicate keys found: 23'); + } + }); + + test('accepts same keys on different level', function () { + const prompt = ( + <> + <> + <Text key={23}>Hola</Text> + </> + <> + <Text key={23}>Adios</Text> + </> + </> + ); + + const reconciler = new VirtualPromptReconciler(prompt); + const result = reconciler.reconcile(); + + const orderedPaths = extractNodesWitPath(result!); + + assert.deepStrictEqual(orderedPaths, [ + '$.f', + '$.f[0].f', + '$.f[0].f["23"].Text', + '$.f[0].f["23"].Text[0]', + '$.f[1].f', + '$.f[1].f["23"].Text', + '$.f[1].f["23"].Text[0]', + ]); + + // Assert uniqueness of paths + assert.deepStrictEqual([...new Set(orderedPaths)], orderedPaths); + }); + + test('Should re-render if the state of the component changed', function () { + let outerShouldRenderChildren: Dispatch<StateUpdater<boolean>>; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [shouldRenderChildren, setShouldRenderChildren] = context.useState(false); + + outerShouldRenderChildren = setShouldRenderChildren; + + if (shouldRenderChildren) { + return <Text>This is my child</Text>; + } + }; + + const reconciler = new VirtualPromptReconciler(<MyTestComponent />); + const resultOne = reconciler.reconcile(); + assert.deepStrictEqual(resultOne!.children?.length, 0); + + outerShouldRenderChildren!(true); + + // Should re-render since the state changed + const resultTwo = reconciler.reconcile(); + assert.deepStrictEqual(resultTwo!.children?.length, 1); + }); + + test('Should re-render if the state of a nested component changed', function () { + let outerSetShouldRenderChildren: Dispatch<StateUpdater<boolean>>; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [shouldRenderChildren, setShouldRenderChildren] = context.useState(false); + + outerSetShouldRenderChildren = setShouldRenderChildren; + + if (shouldRenderChildren) { + return <Text>This is my child</Text>; + } + }; + + const reconciler = new VirtualPromptReconciler( + ( + <> + <MyTestComponent /> + </> + ) + ); + const resultOne = reconciler.reconcile(); + assert.deepStrictEqual(resultOne!.children?.length, 1); + assert.deepStrictEqual(resultOne!.children[0].children?.length, 0); + + outerSetShouldRenderChildren!(true); + + // Should re-render since the state changed + const resultTwo = reconciler.reconcile(); + assert.deepStrictEqual(resultTwo!.children?.length, 1); + assert.deepStrictEqual(resultTwo!.children[0].children?.length, 1); + }); + + test('Should not re-render if the state did not change', function () { + let created = false; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [count, _] = context.useState(0); + + if (created) { + throw new Error('Component was created more than once'); + } + + created = true; + + return <Text>This is my component {count}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<MyTestComponent />); + try { + reconciler.reconcile(); + reconciler.reconcile(); + } catch (e) { + assert.fail('Component was created more than once, which should not happen'); + } + }); + + test('Should preserve child state if position and type within parent are the same', function () { + let outerSetParentState: Dispatch<StateUpdater<string>>; + + const ParentComponent = (props: PromptElementProps, context: ComponentContext) => { + const [parentState, setParentState] = context.useState('BEFORE'); + + outerSetParentState = setParentState; + + return ( + <> + <Text>This is the parent count: {parentState}</Text> + <ChildComponent parentState={parentState} /> + </> + ); + }; + type ChildComponentProps = { parentState: string }; + let childState = 'UNINITIALIZED'; + const ChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childState = childComponentState; + + return <Text>This is the child state {childComponentState}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<ParentComponent />); + + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + + outerSetParentState!('AFTER'); + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + }); + + test('Should not preserve child state if position and type change and switch back', function () { + let outerSetParentState: Dispatch<StateUpdater<string>>; + + const ParentComponent = (props: PromptElementProps, context: ComponentContext) => { + const [parentState, setParentState] = context.useState('BEFORE'); + + outerSetParentState = setParentState; + + if (parentState === 'BEFORE') { + return ( + <> + <Text>This is the parent count: {parentState}</Text> + <ChildComponent parentState={parentState} /> + </> + ); + } + return ( + <> + <ChildComponent parentState={parentState} /> + <Text>This is the parent count: {parentState}</Text> + </> + ); + }; + type ChildComponentProps = { parentState: string }; + let childState = 'UNINITIALIZED'; + const ChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childState = childComponentState; + + return <Text>This is the child state {childComponentState}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<ParentComponent />); + + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + + outerSetParentState!('AFTER'); + reconciler.reconcile(); + assert.strictEqual(childState, 'AFTER'); + + outerSetParentState!('BEFORE'); + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + }); + + test('Should preserve child state if position changes but key stays the same', function () { + let outerSetParentState: Dispatch<StateUpdater<string>>; + + const ParentComponent = (props: PromptElementProps, context: ComponentContext) => { + const [parentState, setParentState] = context.useState('BEFORE'); + + outerSetParentState = setParentState; + + if (parentState === 'BEFORE') { + return ( + <> + <Text>This is the parent count: {parentState}</Text> + <ChildComponent key='child' parentState={parentState} /> + </> + ); + } + return ( + <> + <ChildComponent key='child' parentState={parentState} /> + <Text>This is the parent count: {parentState}</Text> + </> + ); + }; + type ChildComponentProps = { parentState: string }; + let childState = 'UNINITIALIZED'; + const ChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childState = childComponentState; + + return <Text>This is the child state {childComponentState}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<ParentComponent />); + + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + + outerSetParentState!('AFTER'); + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + + outerSetParentState!('BEFORE'); + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + }); + + test('Should preserve child state if position and type within parent are the same with deep nesting', function () { + let outerSetParentState: Dispatch<StateUpdater<string>>; + + const ParentComponent = (props: PromptElementProps, context: ComponentContext) => { + const [parentState, setParentState] = context.useState('BEFORE'); + + outerSetParentState = setParentState; + + return ( + <> + <Text>This is the parent count: {parentState}</Text> + <ChildComponent parentState={parentState} /> + </> + ); + }; + type ChildComponentProps = { parentState: string }; + let childState = 'UNINITIALIZED'; + const ChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childState = childComponentState; + + return ( + <> + <Text>This is the child state {childComponentState}</Text> + <ChildChildComponent parentState={childComponentState} /> + </> + ); + }; + let childChildState = 'UNINITIALIZED'; + const ChildChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childChildState = childComponentState; + + return <Text>This is the child state {childComponentState}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<ParentComponent />); + + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + assert.strictEqual(childChildState, 'BEFORE'); + + outerSetParentState!('AFTER'); + reconciler.reconcile(); + assert.strictEqual(childState, 'BEFORE'); + assert.strictEqual(childChildState, 'BEFORE'); + }); + + test('Should preserve child state if position and type within parent are the same with multiple children of same type', function () { + let outerSetParentState: Dispatch<StateUpdater<string>>; + + const ParentComponent = (props: PromptElementProps, context: ComponentContext) => { + const [parentState, setParentState] = context.useState('BEFORE'); + + outerSetParentState = setParentState; + + return ( + <> + <Text>This is the parent count: {parentState}</Text> + <ChildComponent parentState={parentState + '_A'} /> + <ChildComponent parentState={parentState + '_B'} /> + </> + ); + }; + type ChildComponentProps = { parentState: string }; + let childState: string[] = []; + const ChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childState.push(childComponentState); + + return <Text>This is the child state {childComponentState}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<ParentComponent />); + + reconciler.reconcile(); + assert.deepStrictEqual(childState, ['BEFORE_A', 'BEFORE_B']); + + childState = []; + outerSetParentState!('AFTER'); + reconciler.reconcile(); + assert.deepStrictEqual(childState, ['BEFORE_A', 'BEFORE_B']); + }); + + test('Should initialize child state if position changes on reconciliation', function () { + let outerSetParentCount: Dispatch<StateUpdater<number>>; + let outerSetParentState: Dispatch<StateUpdater<string>>; + + const ParentComponent = (props: PromptElementProps, context: ComponentContext) => { + const [parentState, setParentState] = context.useState('FIRST'); + const [count, setCount] = context.useState(0); + + outerSetParentCount = setCount; + outerSetParentState = setParentState; + + const renderChildren = () => { + const children = []; + for (let i = 0; i < count; i++) { + children.push(<Text>This is the parent count: {parentState}</Text>); + } + children.push(<ChildComponent parentState={parentState} />); + return children; + }; + return <>{renderChildren()}</>; + }; + type ChildComponentProps = { parentState: string }; + let childState = 'UNINITIALIZED'; + const ChildComponent = (props: ChildComponentProps, context: ComponentContext) => { + const [childComponentState, _] = context.useState(props.parentState); + childState = childComponentState; + + return <Text>This is the child state {childComponentState}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<ParentComponent />); + + reconciler.reconcile(); + assert.strictEqual(childState, 'FIRST'); + + outerSetParentCount!(1); + outerSetParentState!('SECOND'); + reconciler.reconcile(); + assert.strictEqual(childState, 'SECOND'); + }); + + test('Should support cancellation', function () { + const cts = new CancellationTokenSource(); + let outerSetCount: Dispatch<StateUpdater<number>> = () => 0; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [count, setCount] = context.useState(0); + outerSetCount = setCount; + return <Text>This is my component {count}</Text>; + }; + + const reconciler = new VirtualPromptReconciler(<MyTestComponent />); + + const result = reconciler.reconcile(cts.token); + outerSetCount(1); + cts.cancel(); + const resultAfterCancellation = reconciler.reconcile(cts.token); + + assert.deepStrictEqual(result, resultAfterCancellation); + }); + + test('Creates a pipe to route data to a component', async function () { + let componentData = ''; + const DataComponent = (props: PromptElementProps, context: ComponentContext) => { + context.useData(isString, (data: string) => { + componentData = data; + }); + return <></>; + }; + const reconciler = new VirtualPromptReconciler(<DataComponent />); + + const pipe = reconciler.createPipe(); + await pipe.pump('test'); + + assert.deepStrictEqual(componentData, 'test'); + }); + + test('Fails to pump data before initialization', async function () { + const reconciler = new VirtualPromptReconciler(undefined as unknown as PromptElement); + const pipe = reconciler.createPipe(); + try { + await pipe.pump('test'); + assert.fail('Should have thrown an error'); + } catch (e) { + assert.equal((e as Error).message, 'No tree to pump data into. Pumping data before initializing?'); + } + }); + + test('Creates a pipe to route data to a component after previous reconciliation has been cancelled', async function () { + const cts = new CancellationTokenSource(); + let componentData = ''; + const DataComponent = (props: PromptElementProps, context: ComponentContext) => { + context.useData(isString, (data: string) => { + componentData = data; + }); + return <></>; + }; + const reconciler = new VirtualPromptReconciler(<DataComponent />); + const pipe = reconciler.createPipe(); + + cts.cancel(); + reconciler.reconcile(cts.token); + await pipe.pump('test'); + + assert.deepStrictEqual(componentData, 'test'); + }); + + test('Computes node statistics on reconcile', async function () { + const DataComponent = (props: PromptElementProps, context: ComponentContext) => { + const [state, setState] = context.useState(''); + context.useData(isString, (data: string) => { + setState(data); + }); + return <>{state}</>; + }; + const reconciler = new VirtualPromptReconciler(<DataComponent />); + + const pipe = reconciler.createPipe(); + await pipe.pump('test'); + const tree = reconciler.reconcile(); + + const updateTime = tree?.lifecycle?.lifecycleData.getUpdateTimeMsAndReset(); + assert.ok(updateTime); + assert.ok(updateTime > 0); + }); + + test('Computes node statistics on reconcile with measurements from data pumping', async function () { + const DataComponent = (props: PromptElementProps, context: ComponentContext) => { + const [state, setState] = context.useState(''); + context.useData(isString, (data: string) => { + setState(data); + }); + return <>{state}</>; + }; + const reconciler = new VirtualPromptReconciler(<DataComponent />); + + const pipe = reconciler.createPipe(); + await pipe.pump('test'); + + let tree = reconciler.reconcile(); + let updateTime = tree?.lifecycle?.lifecycleData.getUpdateTimeMsAndReset(); + assert.ok(updateTime); + assert.ok(updateTime > 0); + + tree = reconciler.reconcile(); + updateTime = tree?.lifecycle?.lifecycleData.getUpdateTimeMsAndReset(); + assert.ok(updateTime === 0); + }); + + test('Updates data time is updated on every data update', async function () { + const DataComponent = (props: PromptElementProps, context: ComponentContext) => { + const [count, setCount] = context.useState(0); + context.useData(isNumber, async (newCount: number) => { + await new Promise(resolve => setTimeout(resolve, count)); + setCount(newCount); + }); + return <>{count}</>; + }; + const reconciler = new VirtualPromptReconciler(<DataComponent />); + const pipe = reconciler.createPipe(); + await pipe.pump(1); + + const tree = reconciler.reconcile(); + const lifeCycleData = tree?.lifecycle?.lifecycleData; + assert.ok(lifeCycleData); + const timeFirstPump = lifeCycleData?.getUpdateTimeMsAndReset(); + assert.ok(timeFirstPump > 0); + await pipe.pump(2); + const timeSecondPump = lifeCycleData?.getUpdateTimeMsAndReset(); + assert.ok(timeSecondPump > 0); + assert.notDeepStrictEqual(timeFirstPump, timeSecondPump); + }); + + test('Creates a pipe to route data to many components', async function () { + let componentDataA = ''; + const DataComponentA = (props: PromptElementProps, context: ComponentContext) => { + context.useData(isString, (data: string) => { + componentDataA = data; + }); + return <></>; + }; + let componentDataB = ''; + const DataComponentB = (props: PromptElementProps, context: ComponentContext) => { + context.useData(isString, (data: string) => { + componentDataB = data; + }); + return <></>; + }; + const reconciler = new VirtualPromptReconciler( + ( + <> + <DataComponentA /> + <DataComponentB /> + </> + ) + ); + + const pipe = reconciler.createPipe(); + await pipe.pump('test'); + + assert.deepStrictEqual(componentDataA, 'test'); + assert.deepStrictEqual(componentDataB, 'test'); + }); + + test('Creates a pipe to route data async to many components', async function () { + let componentDataA = ''; + const DataComponentA = (props: PromptElementProps, context: ComponentContext) => { + context.useData(isString, async (data: string) => { + await Promise.resolve(); + componentDataA = data; + }); + return <></>; + }; + let componentDataB = ''; + const DataComponentB = (props: PromptElementProps, context: ComponentContext) => { + context.useData(isString, async (data: string) => { + await Promise.resolve(); + componentDataB = data; + }); + return <></>; + }; + const reconciler = new VirtualPromptReconciler( + ( + <> + <DataComponentA /> + <DataComponentB /> + </> + ) + ); + + const pipe = reconciler.createPipe(); + await pipe.pump('test'); + + assert.deepStrictEqual(componentDataA, 'test'); + assert.deepStrictEqual(componentDataB, 'test'); + }); + + test('Pumps data to components with any pipe independently', async function () { + const componentDataA: string[] = []; + const DataComponentA = (props: unknown, context: ComponentContext) => { + context.useData(isString, (data: string) => { + componentDataA.push(data); + }); + return <></>; + }; + const componentDataB: string[] = []; + const DataComponentB = (props: unknown, context: ComponentContext) => { + context.useData(isString, (data: string) => { + componentDataB.push(data); + }); + return <></>; + }; + const reconciler = new VirtualPromptReconciler( + ( + <> + <DataComponentA /> + <DataComponentB /> + </> + ) + ); + + const pipe1 = reconciler.createPipe(); + await pipe1.pump('test'); + const pipe2 = reconciler.createPipe(); + await pipe2.pump('test2'); + + assert.deepStrictEqual(componentDataA, ['test', 'test2']); + assert.deepStrictEqual(componentDataB, ['test', 'test2']); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/components/testHelpers.ts b/completions-sample-code/vscode-node/prompt/src/test/components/testHelpers.ts new file mode 100644 index 0000000..f238465 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/components/testHelpers.ts @@ -0,0 +1,22 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PromptSnapshotNode } from '../../components/components'; +import { VirtualPromptNode } from '../../components/reconciler'; + +export function extractNodesWitPath(node: VirtualPromptNode | PromptSnapshotNode): string[] { + if (node.children === undefined || node.children.length === 0) { + return [node.path]; + } + return [node.path, ...(node.children?.flatMap(extractNodesWitPath) ?? [])]; +} + +export function isString(value: unknown): value is string { + return typeof value === 'string'; +} + +export function isNumber(value: unknown): value is number { + return typeof value === 'number'; +} diff --git a/completions-sample-code/vscode-node/prompt/src/test/components/virtualPrompt.test.tsx b/completions-sample-code/vscode-node/prompt/src/test/components/virtualPrompt.test.tsx new file mode 100644 index 0000000..2e44646 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/components/virtualPrompt.test.tsx @@ -0,0 +1,175 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** @jsxRuntime automatic */ +/** @jsxImportSource ../../../jsx-runtime */ +import { + ComponentContext, + PromptElement, + PromptElementProps, + PromptSnapshotNode, + Text, +} from '../../components/components'; +import { Dispatch, StateUpdater } from '../../components/hooks'; +import { VirtualPrompt } from '../../components/virtualPrompt'; +import * as assert from 'assert'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; + +suite('Virtual prompt', function () { + test('The virtual prompt should return a snapshot tree of a prompt', function () { + const prompt = ( + <> + <Text>This is text</Text> + <Text>This is more text</Text> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + const { snapshot } = virtualPrompt.snapshot(); + const nodeNames = getNodeNames(snapshot!); + + const expected = { + name: 'f', + children: [ + { + name: 'Text', + children: [ + { + name: 'string', + children: [], + }, + ], + }, + { + name: 'Text', + children: [ + { + name: 'string', + children: [], + }, + ], + }, + ], + }; + + assert.deepStrictEqual(nodeNames, expected); + }); + + test('The virtual prompt should return an updated snapshot if the inner state changed', function () { + let outerSetCount: Dispatch<StateUpdater<number>>; + let renderCount = 0; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [count, setCount] = context.useState(0); + + outerSetCount = setCount; + renderCount++; + + return <Text>This is my component {count}</Text>; + }; + + const virtualPrompt = new VirtualPrompt(<MyTestComponent />); + const { snapshot: snapshotOne } = virtualPrompt.snapshot(); + + outerSetCount!(1); + + const { snapshot: snapshotTwo } = virtualPrompt.snapshot(); + + assert.strictEqual(renderCount, 2); + assert.notDeepStrictEqual(snapshotOne, snapshotTwo); + }); + + test('Should cancel while snapshotting', function () { + let shouldCancel = false; + let outerCancelCount: Dispatch<StateUpdater<number>>; + const cts = new CancellationTokenSource(); + + const CancellingComponent = (props: PromptElementProps, context: ComponentContext) => { + const [_, setCount] = context.useState(0); + outerCancelCount = setCount; + + // Cancel on second rendering + if (shouldCancel) { + cts.cancel(); + } + shouldCancel = true; + return <Text>CancellingComponent</Text>; + }; + const prompt = ( + <> + <CancellingComponent /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + + outerCancelCount!(1); + + const result = virtualPrompt.snapshot(cts.token); + + assert.deepStrictEqual(result, { snapshot: undefined, status: 'cancelled' }); + }); + + test('Should return an error if there was an error during snapshot', function () { + const virtualPrompt = new VirtualPrompt(undefined as unknown as PromptElement); + + const result = virtualPrompt.snapshot(); + + assert.deepStrictEqual(result.snapshot, undefined); + assert.deepStrictEqual(result.status, 'error'); + assert.deepStrictEqual(result.error?.message, 'No tree to reconcile, make sure to pass a valid prompt'); + }); + + test('Should return an error if there was an error during reconciliation', function () { + let outerSetCount: Dispatch<StateUpdater<number>>; + let created = false; + + const MyTestComponent = (props: PromptElementProps, context: ComponentContext) => { + const [count, setCount] = context.useState(0); + + if (created) { + throw new Error('Component was recreated'); + } + + created = true; + outerSetCount = setCount; + + return <Text>This is my component {count}</Text>; + }; + + const prompt = ( + <> + <MyTestComponent /> + </> + ); + + const virtualPrompt = new VirtualPrompt(prompt); + + outerSetCount!(1); + + const result = virtualPrompt.snapshot(); + + assert.deepStrictEqual(result.snapshot, undefined); + assert.deepStrictEqual(result.status, 'error'); + assert.deepStrictEqual(result.error?.message, 'Component was recreated'); + }); + + test('Should create a pipe', function () { + const virtualPrompt = new VirtualPrompt(<>test</>); + + const pipe = virtualPrompt.createPipe(); + + assert.ok(pipe); + }); +}); + +type NodeName = { name: string; children: NodeName[] }; + +function getNodeNames(node: PromptSnapshotNode): NodeName { + return { + name: node.name, + children: node.children?.map(getNodeNames) ?? [], + }; +} diff --git a/completions-sample-code/vscode-node/prompt/src/test/components/walker.test.ts b/completions-sample-code/vscode-node/prompt/src/test/components/walker.test.ts new file mode 100644 index 0000000..a8a0584 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/components/walker.test.ts @@ -0,0 +1,233 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PromptSnapshotNode } from '../../components/components'; +import { SnapshotWalker } from '../../components/walker'; +import * as assert from 'assert'; + +suite('Snapshot Walker', function () { + test('walks snapshot recursively', function () { + const snapshot = createTestSnapshot(1, 1); + const walker = new SnapshotWalker(snapshot); + const visitedValues: string[] = []; + + walker.walkSnapshot((node, parent, context) => { + visitedValues.push(node.path ?? 'undefined'); + return true; + }); + + assert.deepStrictEqual(visitedValues, ['0', '0.0']); + }); + + test('stops walking after visitor returns false', function () { + const snapshot = createTestSnapshot(2, 2); + const walker = new SnapshotWalker(snapshot); + const visitedPaths: string[] = []; + + walker.walkSnapshot((node, parent, context) => { + visitedPaths.push(node.path); + return false; + }); + + assert.deepStrictEqual(visitedPaths, ['0']); + }); + + test('walks deeper nested snapshot', function () { + const snapshot = createTestSnapshot(3, 2); + const walker = new SnapshotWalker(snapshot); + const paths: string[] = []; + + walker.walkSnapshot((node, parent, context) => { + paths.push(node.path); + return true; + }); + + assert.deepStrictEqual(paths, [ + '0', + '0.0', + '0.0.0', + '0.0.0.0', + '0.0.0.1', + '0.0.1', + '0.0.1.0', + '0.0.1.1', + '0.1', + '0.1.0', + '0.1.0.0', + '0.1.0.1', + '0.1.1', + '0.1.1.0', + '0.1.1.1', + ]); + }); + + test('carries weight relative to parent weight', function () { + const snapshot: PromptSnapshotNode = { + name: 'root', + path: '0', + value: '0', + props: { weight: 0.5 }, + children: [ + { + name: 'child', + path: '0.0', + value: '1', + props: { weight: 0.5 }, + statistics: {}, + }, + ], + statistics: {}, + }; + + const walker = new SnapshotWalker(snapshot); + const weights: number[] = []; + + walker.walkSnapshot((node, parent, context) => { + weights.push(context.weight as number); + return true; + }); + + assert.deepStrictEqual(weights, [0.5, 0.25]); // root: 0.5, child: 0.5 * 0.5 + }); + + test('propagates chunks to children', function () { + const snapshot: PromptSnapshotNode = { + name: 'Chunk', + path: '0', + value: 'chunk1', + statistics: {}, + children: [ + { + name: 'child', + path: '0.0', + value: 'child1', + statistics: {}, + }, + ], + }; + + const walker = new SnapshotWalker(snapshot); + const chunks: Set<string>[] = []; + + walker.walkSnapshot((node, parent, context) => { + chunks.push(context.chunks as Set<string>); + return true; + }); + + assert.deepStrictEqual(chunks.length, 2); + + const chunk = new Set<string>(['0']); + assert.deepStrictEqual(chunks[0], chunk); + assert.deepStrictEqual(chunks[1], chunk); + }); + + test('propagates nested chunks', function () { + const snapshot: PromptSnapshotNode = { + name: 'Chunk', + path: '0', + value: 'chunk1', + statistics: {}, + children: [ + { + name: 'child', + path: '0.0', + value: 'child1', + statistics: {}, + }, + { + name: 'Chunk', + path: '0.1', + value: 'chunk2', + statistics: {}, + children: [ + { + name: 'child', + path: '0.1.0', + value: 'child2', + statistics: {}, + }, + ], + }, + ], + }; + + const walker = new SnapshotWalker(snapshot); + const chunks: Set<string>[] = []; + + walker.walkSnapshot((node, parent, context) => { + chunks.push(context.chunks as Set<string>); + return true; + }); + + assert.deepStrictEqual(chunks.length, 4); + + const chunk = new Set<string>(['0']); + const nestedChunk = new Set<string>(['0', '0.1']); + assert.deepStrictEqual(chunks[0], chunk); + assert.deepStrictEqual(chunks[1], chunk); + assert.deepStrictEqual(chunks[2], nestedChunk); + assert.deepStrictEqual(chunks[3], nestedChunk); + }); + + test('propagates source to children', function () { + const snapshot: PromptSnapshotNode = { + name: 'root', + path: '0', + value: 'root', + props: { source: 'source1' }, + statistics: {}, + children: [ + { + name: 'child', + path: '0.0', + value: 'child', + statistics: {}, + }, + ], + }; + + const walker = new SnapshotWalker(snapshot); + const sources: unknown[] = []; + + walker.walkSnapshot((node, parent, context) => { + sources.push(context.source); + return true; + }); + + assert.deepStrictEqual(sources, ['source1', 'source1']); + }); + + function createTestSnapshot( + depth: number, + childrenCount: number = 3, + currentPath: string = '' + ): PromptSnapshotNode { + if (depth <= 0) { + return { + name: 'leaf', + path: currentPath || '0', + value: currentPath || '0', + statistics: {}, + }; + } + + const children: PromptSnapshotNode[] = []; + const nodeIndex = currentPath || '0'; + + // Create configurable number of children at each level + for (let i = 0; i < childrenCount; i++) { + const childPath = `${nodeIndex}.${i}`; + children.push(createTestSnapshot(depth - 1, childrenCount, childPath)); + } + + return { + name: `node-${nodeIndex}`, + path: nodeIndex, + value: nodeIndex, + children, + statistics: {}, + }; + } +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/indentation.test.ts b/completions-sample-code/vscode-node/prompt/src/test/indentation.test.ts new file mode 100644 index 0000000..0282df4 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/indentation.test.ts @@ -0,0 +1,453 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { + blankNode, + clearLabels, + clearLabelsIf, + cutTreeAfterLine, + deparseAndCutTree, + deparseLine, + deparseTree, + duplicateTree, + firstLineOf, + foldTree, + IndentationTree, + isBlank, + isLine, + lastLineOf, + lineNode, + LineNode, + mapLabels, + parseRaw, + parseTree, + resetLineNumbers, + topNode, + virtualNode, + visitTree, + visitTreeConditionally, +} from '../indentation'; +import { compareTreeWithSpec } from './testHelpers'; + +function doParseTest<T>(source: string, expectedTree: IndentationTree<T>) { + const tree = clearLabels(parseTree(source, 'python')); + compareTreeWithSpec(tree, expectedTree); +} + +const SOURCE = { + source: dedent` +f1: + a1 +f2: + a2 + a3 +`, + name: '', +}; + +suite('Test compareTreeWithSpec', function () { + const SOURCE_MISSING_CHILD = { + source: dedent` +f1: + a1 +f2: + a2 +`, + name: 'missing child', + }; + + const SOURCE_EXTRA_CHILD = { + source: dedent` +f1: + a1 +f2: + a2 + a3 + a4 +`, + name: 'extra_child', + }; + + const SOURCE_MISSING_SIBLING = { + source: dedent` +f1: + a1 +`, + name: 'missing sibling', + }; + + const SOURCE_EXTRA_SIBLING = { + source: dedent` +f1: + a1 +f2: + a2 + a3 +f3: + a4 +`, + name: 'extra_sibling', + }; + + const SOURCE_EXTRA_MIDDLE_BLANK_LINE = { + source: dedent` +f1: + a1 + +f2: + a2 + a3 +`, + name: 'extra middle blank line', + }; + + const SOURCE_EXTRA_TRAILING_BLANK_LINE = { + source: dedent` +f1: + a1 +f2: + a2 + a3 + +`, + name: 'extra trailing blank line', + }; + + const SOURCE_EXTRA_INDENTATION = { + source: dedent` +f1: + a1 +f2: + a2 + a3 +`, + name: 'extra indentation', + }; + + const expected = topNode([ + lineNode(0, 0, 'f1:', [lineNode(4, 1, 'a1', [])]), + lineNode(0, 2, 'f2:', [lineNode(4, 3, 'a2', []), lineNode(4, 4, 'a3', [])]), + ]); + + test('Test compareTreeWithSpec with good input', function () { + doParseTest(SOURCE.source, expected); + }); + // Loop over all bad inputs where we expect a failure from compareTreeWithSpec + for (const badInput of [ + SOURCE_MISSING_CHILD, + SOURCE_EXTRA_CHILD, + SOURCE_MISSING_SIBLING, + SOURCE_EXTRA_SIBLING, + SOURCE_EXTRA_INDENTATION, + SOURCE_EXTRA_TRAILING_BLANK_LINE, + ]) { + test(`Test compareTreeWithSpec with bad input ${badInput.name}`, function () { + assert.throws( + () => doParseTest(badInput.source, expected), + assert.AssertionError, + `Expected to fail with ${JSON.stringify(badInput)}` + ); + }); + } + + // Do we want extra blank lines to be children? + test('Test compareTreeWithSpec with extra blank line input', function () { + assert.throws( + () => doParseTest(SOURCE_EXTRA_MIDDLE_BLANK_LINE.source, expected), + assert.AssertionError, + 'Expected to fail with extra blank line, actually fails with extra child' + ); + }); +}); + +suite('Tree core functions: label manipulation', function () { + function setOfLabels<L>(tree: IndentationTree<L>): Set<L | 'undefined'> { + const labels = new Set<L | 'undefined'>(); + visitTree( + tree, + node => { + labels.add(node.label ?? 'undefined'); + }, + 'topDown' + ); + return labels; + } + test('Remove labels from tree', function () { + const tree = parseTree(SOURCE.source, 'python'); + setOfLabels(tree); + visitTree( + tree, + node => { + node.label = node.type === 'line' && node.lineNumber % 2 === 0 ? 'foo' : 'bar'; + }, + 'topDown' + ); + setOfLabels(tree); + assert.notDeepStrictEqual([...setOfLabels(tree)], ['undefined'], 'Tree never had labels'); + clearLabels(tree); + assert.deepStrictEqual([...setOfLabels(tree)], ['undefined'], 'Tree still has labels'); + }); + test('Remove certain labels from tree', function () { + const tree = parseRaw(SOURCE.source) as IndentationTree<string>; + visitTree( + tree, + node => { + node.label = node.type === 'line' && node.lineNumber % 2 === 0 ? 'foo' : 'bar'; + }, + 'topDown' + ); + assert.deepStrictEqual([...setOfLabels(tree)], ['bar', 'foo'], 'Did not prepare tree as expected'); + clearLabelsIf<'foo', 'bar'>( + tree as IndentationTree<'foo' | 'bar'>, + // type predicate of form arg is 'bar': + (arg: 'foo' | 'bar'): arg is 'bar' => arg === 'bar' + ); + assert.deepStrictEqual([...setOfLabels(tree)], ['undefined', 'foo'], 'Did not remove bar labels'); + }); + test('Test mapLabels', function () { + const tree = parseTree(SOURCE.source + '\n\nprint("bye")', 'python'); + visitTree( + tree, + node => { + node.label = node.type === 'line' && node.lineNumber % 2 === 0 ? 'foo' : 'bar'; + }, + 'topDown' + ); + assert.deepStrictEqual([...setOfLabels(tree)], ['bar', 'foo'], 'Did not prepare tree as expected'); + const labelsBefore = foldTree(tree, [] as string[], (node, acc) => [...acc, node.label ?? ''], 'topDown'); + const mapfct = (label: string) => (label === 'foo' ? 1 : 2); + const treeWithNumbers = mapLabels(tree as IndentationTree<'foo' | 'bar'>, mapfct); + const labelsAfter = foldTree( + treeWithNumbers, + [] as Array<string | number>, + (node, acc) => [...acc, node.label ?? ''], + 'topDown' + ); + assert.deepStrictEqual([...setOfLabels(treeWithNumbers)], [2, 1], 'Did not map labels'); + assert.deepStrictEqual(labelsBefore.map(mapfct), labelsAfter, 'Did not map labels right'); + }); +}); + +suite('Tree core functions: line numbers', function () { + const tree = parseTree(SOURCE.source, 'python'); + test('First line of source tree is 0', function () { + assert.strictEqual(firstLineOf(tree), 0); + }); + test('First line of source tree + two newlines is 2', function () { + const offsetTree = parseTree(`\n\n${SOURCE.source}`, 'python'); + const originalTree = offsetTree.subs[2]; + assert.strictEqual(firstLineOf(originalTree), 2); + }); + test('Last line of source tree is 4', function () { + assert.strictEqual(lastLineOf(tree), 4); + }); + test('firstLineOf', function () { + const firstLine = firstLineOf( + topNode([virtualNode(0, []), virtualNode(0, [lineNode(0, 5, 'zero', [])]), lineNode(0, 6, 'one', [])]) + ); + assert.ok(firstLine !== undefined); + assert.strictEqual(firstLine, 5); + }); + test('firstLineOf undefined', function () { + const firstLine = firstLineOf(topNode([virtualNode(0, []), virtualNode(0, [virtualNode(0, [])])])); + assert.ok(firstLine === undefined); + }); + test('firstLineOf blank', function () { + const firstLine = firstLineOf(topNode([blankNode(1), lineNode(0, 2, 'line', [])])); + assert.ok(firstLine === 1); + }); + test('lastLineOf', function () { + const line = lastLineOf( + topNode([ + virtualNode(0, []), + virtualNode(0, [lineNode(0, 1, 'first', [])]), + lineNode(0, 2, 'second', [lineNode(0, 3, 'third', []), lineNode(0, 4, 'fourth', [])]), + ]) + ); + assert.ok(line !== undefined); + assert.strictEqual(line, 4); + }); + test('lastLineOf take by tree order, not registered line numbers', function () { + const line = lastLineOf( + topNode([ + lineNode( + 0, + 5, + 'parent', + [lineNode(0, 4, 'child 1', []), lineNode(0, 3, 'child 2', []), lineNode(0, 2, 'child 3', [])], + 5 + ), + ]) + ); + assert.ok(line !== undefined); + assert.strictEqual(line, 2); + }); + test('lastLineOf undefined', function () { + const line = lastLineOf(topNode([virtualNode(0, []), virtualNode(0, [virtualNode(0, [])])])); + assert.ok(line === undefined); + }); + + test('lastLineOf blank', function () { + const line = lastLineOf(topNode([lineNode(0, 1, 'line', []), blankNode(2)])); + assert.ok(line === 2); + }); + test('Reset line numbers for tree', function () { + const duplicatedTree = duplicateTree(tree); + visitTree( + duplicatedTree, + node => { + if (isLine(node)) { node.lineNumber = -1; } + }, + 'topDown' + ); + assert.strictEqual(firstLineOf(duplicatedTree), -1); + assert.strictEqual(lastLineOf(duplicatedTree), -1); + resetLineNumbers(duplicatedTree); + let counter = 0; + visitTree( + duplicatedTree, + node => { + if (isLine(node) || isBlank(node)) { + assert.strictEqual(node.lineNumber, counter); + counter++; + } + }, + 'topDown' + ); + }); +}); + +suite('Test core functions: other', function () { + const tree = parseTree(SOURCE.source, 'python'); + test('deparseTree should give same output as source input', function () { + // Assert that the tree is the same as the source, ignoring trailing newlines + assert.strictEqual(deparseTree(tree).replace(/\n*$/, ''), SOURCE.source.replace(/\n*$/, '')); + }); + test('deparseTree should give same output as source input with an extra blank line', function () { + const treeLonger = parseTree(`${SOURCE.source}\n`, 'python'); + // Assert that the tree is the same as the source, ignoring trailing newlines + assert.strictEqual(deparseTree(treeLonger).replace(/\n*$/, ''), SOURCE.source.replace(/\n*$/, '')); + }); + test('deparseAndCutTree cuts at labels', function () { + const source = dedent` +1 + 2 + 3 +4 + 5 + 6 +7 + 8 + 9`; + const tree = parseRaw(source) as IndentationTree<string>; + tree.subs[0].subs[1].label = 'cut'; + tree.subs[1].subs[0].label = 'cut'; + const cuts = deparseAndCutTree(tree, ['cut']); + // since there were two cuts, it's cut in 5 bits: + assert.strictEqual(cuts.length, 5); + // it's cut at the lines labeled 'cut' + assert.strictEqual(cuts[1].source, deparseLine(tree.subs[0].subs[1] as LineNode<string>)); + assert.strictEqual(cuts[3].source, deparseLine(tree.subs[1].subs[0] as LineNode<string>)); + // all together give the original source (ignoring trailing newlines -- _all_ cuts are newline ended) + assert.strictEqual(cuts.map(x => x.source).join(''), source + '\n'); + }); + /* test('encodeTree should give an expression coding the tree', function () { + const source = dedent` + 1 + 2 + 3 + + 4 ( + 5 + 6 + ) + + + 7 + 8 + 9 + )`; + const tree = groupBlocks(parseTree(source)); + // to eval, need to make several imports explicit + const functions = [topNode, virtualNode, lineNode, blankNode]; + assert.notStrictEqual(functions, []); // make functions used + const treeAfterRoundTrip = <IndentationTree<string>>eval(` + const topNode = functions[0]; + const virtualNode = functions[1]; + const lineNode = functions[2]; + const blankNode = functions[3]; + ${encodeTree(tree)}`); + compareTreeWithSpec(treeAfterRoundTrip, tree); + }); */ + test('Cutting tree correctly', function () { + const cutTree = parseTree(SOURCE.source, 'python'); + cutTreeAfterLine(cutTree, 2); + assert.strictEqual(lastLineOf(cutTree), 2); + }); + test('VisitTreeConditionally', function () { + const tree = parseRaw(dedent` +1 + 2 + 3 +4 + 5 + 6 +7 + 8 + 9`); + const traceTopDownAll: string[] = []; + visitTree( + tree, + node => { + if (node.type === 'line') { traceTopDownAll.push(node.sourceLine.trim()); } + return node.type === 'top'; + }, + 'topDown' + ); + assert.deepStrictEqual( + traceTopDownAll, + ['1', '2', '3', '4', '5', '6', '7', '8', '9'], + 'visit all in order: top to down' + ); + + const traceButtonUpAll: string[] = []; + visitTree( + tree, + node => { + if (node.type === 'line') { traceButtonUpAll.push(node.sourceLine.trim()); } + return node.type === 'top'; + }, + 'bottomUp' + ); + assert.deepStrictEqual( + traceButtonUpAll, + ['2', '3', '1', '5', '6', '4', '8', '9', '7'], + 'visit all in order: first leaves, then parents' + ); + + const traceTopDown: string[] = []; + visitTreeConditionally( + tree, + node => { + if (node.type === 'line') { traceTopDown.push(node.sourceLine.trim()); } + return traceTopDown.length < 4; + }, + 'topDown' + ); + assert.deepStrictEqual(traceTopDown, ['1', '2', '3', '4'], 'should stop after four lines'); + + const traceButtomUp: string[] = []; + visitTreeConditionally( + tree, + node => { + if (node.type === 'line') { traceButtomUp.push(node.sourceLine.trim()); } + return traceButtomUp.length < 4; + }, + 'bottomUp' + ); + assert.deepStrictEqual(traceButtomUp, ['2', '3', '1', '5'], 'should stop after four nodes'); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/indentationLanguages.test.ts b/completions-sample-code/vscode-node/prompt/src/test/indentationLanguages.test.ts new file mode 100644 index 0000000..b1ff0b4 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/indentationLanguages.test.ts @@ -0,0 +1,265 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { blankNode, isLine, lineNode, parseTree, topNode, virtualNode, visitTree } from '../indentation'; +import { compareTreeWithSpec } from './testHelpers'; + +/** Test some language specific parsing techniques */ +suite('Java', function () { + test('method detection in Java', function () { + const source = dedent` + // first an import + import java.util.List; + + @Override + public class Test { + public static void main(String[] args) { + System.out.println("Hello World!"); + + } + + @Override + private List<String> list; + }`; + const javaParsedTree = parseTree(source, 'java'); + + // we should have picked up the correct labels + const lineLabels: string[] = []; + visitTree( + javaParsedTree, + node => { + if (isLine(node) && node.label) { + lineLabels.push(node.label); + } + }, + 'topDown' + ); + assert.deepStrictEqual(lineLabels, [ + 'comment_single', + 'import', + // blank + 'annotation', + 'class', + 'member', + // not labelled + 'closer', + // blank + 'member', // as per explicit comment, the annotations within a class are relabeled 'member, + 'member', + 'closer', + ]); + }); + + test('labelLines java', function () { + const tree = parseTree( + dedent` +package com.example; +import java.awt.*; +@annotation +final public class A { + /** A javadoc + * Second line + */ + public static void main(String[] args) { + // single-line comment + /* Multiline + * comment + */ + System.out.println("Hello, world!"); + } +} +public interface I { } +`, + 'java' + ); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'pa...', [], 'package'), + lineNode(0, 1, 'imp..', [], 'import'), + lineNode(0, 2, '@ann...', [], 'annotation'), + lineNode( + 0, + 3, + 'cla...', + [ + lineNode(4, 4, '/**...', [lineNode(5, 5, '* ...', []), lineNode(5, 6, '* ...', [])], 'javadoc'), + lineNode(4, 7, 'public...', [ + lineNode(8, 8, '//...', [], 'comment_single'), + lineNode( + 8, + 9, + '/*...', + [lineNode(9, 10, '* ...', []), lineNode(9, 11, '*/', [])], + 'comment_multi' + ), + lineNode(8, 12, 'System ...', []), + lineNode(4, 13, '}', [], 'closer'), + ]), + lineNode(0, 14, '}', [], 'closer'), + ], + 'class' + ), + lineNode(0, 15, 'public...', [], 'interface'), + ]) + ); + }); + + test('parse Java fields', function () { + //TODO: Add a field with annotation on separate line + const tree = parseTree( + dedent` +class A { + int a; + /** Javadoc */ + int b; + // Comment + @Native int c; +} +`, + 'java' + ); + compareTreeWithSpec( + tree, + topNode([ + lineNode( + 0, + 0, + 'class...', + [ + lineNode(4, 1, 'int a;', [], 'member'), + lineNode(4, 2, '/**...', [], 'javadoc'), + lineNode(4, 3, 'int b;', [], 'member'), + lineNode(4, 4, '//...', [], 'comment_single'), + lineNode(4, 5, '@Native int c;', [], 'member'), + lineNode(0, 6, '}', [], 'closer'), + ], + 'class' + ), + ]) + ); + }); + + test('parse Java inner class', function () { + const tree = parseTree( + dedent` +class A { + int a; + + class Inner { + int b; + } + + interface InnerInterface { + int myMethod(); + } +} +`, + 'java' + ); + compareTreeWithSpec( + tree, + topNode([ + lineNode( + 0, + 0, + 'class A {', + [ + lineNode(4, 1, 'int a;', [], 'member'), + blankNode(2), + lineNode( + 4, + 3, + 'class Inner ...', + [lineNode(8, 4, 'int b;', [], 'member'), lineNode(4, 5, '}', [], 'closer')], + 'class' + ), + blankNode(6), + lineNode( + 4, + 7, + 'interface InnerInterface ...', + [lineNode(8, 8, 'int myMethod();', [], 'member'), lineNode(4, 9, '}', [], 'closer')], + 'interface' + ), + lineNode(0, 10, '}', [], 'closer'), + ], + 'class' + ), + ]) + ); + }); +}); + +suite('Markdown', function () { + test('header processing in markdown', function () { + const source = dedent` +A + +# B +C +D + +## E +F +G + +# H +I + +### J +K + +L +M +`; + const mdParsedTree = parseTree(source, 'markdown'); + + compareTreeWithSpec( + mdParsedTree, + topNode([ + virtualNode(0, [lineNode(0, 0, 'A', []), blankNode(1)]), + virtualNode(0, [ + lineNode( + 0, + 2, + '# B', + [ + virtualNode(0, [lineNode(0, 3, 'C', []), lineNode(0, 4, 'D', []), blankNode(5)]), + lineNode( + 0, + 6, + '## E', + [lineNode(0, 7, 'F', []), lineNode(0, 8, 'G', []), blankNode(9)], + 'subheading' + ), + ], + 'heading' + ), + lineNode( + 0, + 10, + '# H', + [ + virtualNode(0, [lineNode(0, 11, 'I', []), blankNode(12)]), + lineNode( + 0, + 13, + '### J', + [ + virtualNode(0, [lineNode(0, 14, 'K', []), blankNode(15)]), + virtualNode(0, [lineNode(0, 16, 'L', []), lineNode(0, 17, 'M', [])]), + ], + 'subsubheading' + ), + ], + 'heading' + ), + ]), + ]) + ); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/indentationParsing.test.ts b/completions-sample-code/vscode-node/prompt/src/test/indentationParsing.test.ts new file mode 100644 index 0000000..ee0591a --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/indentationParsing.test.ts @@ -0,0 +1,656 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { + blankNode, + buildLabelRules, + combineClosersAndOpeners, + flattenVirtual, + groupBlocks, + IndentationSubTree, + IndentationTree, + isLine, + isVirtual, + labelLines, + lineNode, + parseRaw, + parseTree, + topNode, + VirtualNode, + virtualNode, + visitTree, +} from '../indentation'; +import { compareTreeWithSpec } from './testHelpers'; + +/** + * Parse a tree according to indentation, where lines + * with content "-> virtual" are translated into virtual nodes + * E.g. + * A + * -> virtual + * B + * C + * Will be parsed as: A having a virtual child, whose children are B and C + * @param sourceParsedAsIf + * @returns + */ +function parseAsIfVirtual(sourceParsedAsIf: string) { + const treeExpected = parseRaw(sourceParsedAsIf); + visitTree( + treeExpected, + node => { + if (isLine(node) && node.sourceLine.trim() === '-> virtual') { + node = node as unknown as VirtualNode<never>; + node.type = 'virtual'; + } + }, + 'topDown' + ); + return treeExpected; +} + +suite('Test core parsing elements', function () { + test('flattenVirtual 1', function () { + const before = topNode([virtualNode(0, []), virtualNode(0, [lineNode(0, 0, 'lonely node', [])])]); + const after = topNode([lineNode(0, 0, 'lonely node', [])]); + compareTreeWithSpec(flattenVirtual(before), after); + }); + + test('flattenVirtual 2', function () { + const before = topNode([lineNode(0, 0, 'A', [virtualNode(2, [lineNode(2, 1, 'lonely node', [])])])]); + const after = topNode([lineNode(0, 0, 'A', [lineNode(2, 1, 'lonely node', [])])]); + compareTreeWithSpec(flattenVirtual(before), after); + }); + + test('groupBlocks basic cases', function () { + const source = dedent` +A + +B +C +D + +E +F + +G +H`; + const tree = parseRaw(source); + const blockTree = groupBlocks(tree); + function assertChildrenAreTheFollowingLines( + tree: IndentationTree<never>, + children: (string | number)[], + message: string = '' + ) { + assert.deepStrictEqual( + tree.subs.map((node: IndentationSubTree<string>) => (isVirtual(node) ? 'v' : node.lineNumber)), + children, + message + ); + } + assertChildrenAreTheFollowingLines(blockTree, ['v', 'v', 'v', 'v'], 'wrong topline blocks'); + assertChildrenAreTheFollowingLines(blockTree.subs[0], [0, 1], 'wrong zeroth block'); + assertChildrenAreTheFollowingLines(blockTree.subs[1], [2, 3, 4, 5], 'wrong first block'); + assertChildrenAreTheFollowingLines(blockTree.subs[2], [6, 7, 8], 'wrong second block'); + assertChildrenAreTheFollowingLines(blockTree.subs[3], [9, 10], 'wrong fourth block'); + }); + + test('groupBlocks advanced cases', function () { + // tests consecutive blank lines, first child blank lines, + // blank lines after last child, lone blank lines, + // consecutive lone blank lines, offside blocks + let tree = parseRaw(dedent` +A + + B + C + D + + E + + + F + +G + H + I + J + + K +`); + tree = groupBlocks(tree); + compareTreeWithSpec( + tree, + topNode([ + virtualNode(0, [ + lineNode(0, 0, 'A', [ + blankNode(1), + virtualNode(2, [ + lineNode(2, 2, 'B', []), + lineNode(2, 3, 'C', [lineNode(4, 4, 'D', [])]), + blankNode(5), + ]), + virtualNode(2, [lineNode(2, 6, 'E', []), blankNode(7), blankNode(8)]), + virtualNode(2, [lineNode(2, 9, 'F', [])]), + ]), + blankNode(10), + ]), + virtualNode(0, [ + lineNode(0, 11, 'G', [ + virtualNode(4, [ + lineNode(4, 12, 'H', []), + lineNode(4, 13, 'I', []), + lineNode(2, 14, 'J', []), + blankNode(15), + ]), + virtualNode(4, [lineNode(2, 16, 'K', [])]), + ]), + ]), + ]) + ); + }); + + test('groupBlocks consecutive blanks as oldest children', function () { + let tree = parseRaw(dedent` +A + + + B1 + B2 +C +`); + tree = groupBlocks(tree); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'A', [ + blankNode(1), + blankNode(2), + virtualNode(4, [lineNode(4, 3, 'B1', []), lineNode(4, 4, 'B2', [])]), + ]), + lineNode(0, 5, 'C', []), + ]) + ); + }); + + test('groupBlocks subs ending with a blank line', function () { + const baseTree = topNode([ + lineNode(0, 0, 'A', [blankNode(1)]), + lineNode(0, 2, 'B', [blankNode(3), blankNode(4)]), + blankNode(5), + lineNode(0, 6, 'C', []), + ]); + const tree = groupBlocks(baseTree); + compareTreeWithSpec( + tree, + topNode([ + virtualNode(0, [ + lineNode(0, 0, 'A', [blankNode(1)]), + lineNode(0, 2, 'B', [blankNode(3), blankNode(4)]), + blankNode(5), + ]), + virtualNode(0, [lineNode(0, 6, 'C', [])]), + ]) + ); + }); + + test('groupBlocks with different delimiter', function () { + let tree = parseRaw(dedent` +A +B +C +D +E +`) as IndentationTree<string>; + const isDelimiter = (node: IndentationTree<string>) => + isLine(node) && (node.sourceLine.trim() === 'B' || node.sourceLine.trim() === 'D'); + tree = groupBlocks(tree, isDelimiter); + compareTreeWithSpec( + tree, + topNode([ + virtualNode(0, [lineNode(0, 0, 'A', []), lineNode(0, 1, 'B', [])]), + virtualNode(0, [lineNode(0, 2, 'C', []), lineNode(0, 3, 'D', [])]), + virtualNode(0, [lineNode(0, 4, 'E ', [])]), + ]) + ); + }); +}); + +suite('Raw parsing', function () { + test('parseRaw', function () { + compareTreeWithSpec( + parseRaw(dedent` +A + a +B + b1 + b2 +C + c1 + c2 + c3 +D + d1 + d2 +`), + topNode([ + lineNode(0, 0, 'A', [lineNode(2, 1, 'a', [])]), + lineNode(0, 2, 'B', [lineNode(2, 3, 'b1', []), lineNode(2, 4, 'b2', [])]), + lineNode(0, 5, 'C', [lineNode(4, 6, 'c1', []), lineNode(4, 7, 'c2', []), lineNode(2, 8, 'c3', [])]), + lineNode(0, 9, 'D', [lineNode(2, 10, 'd1', [lineNode(4, 11, 'd2', [])])]), + ]) + ); + }); + + test('parseRaw blanks', function () { + compareTreeWithSpec( + parseRaw(dedent` +E + e1 + + e2 +F + + f1 +G + g1 + +H + +`), + topNode([ + lineNode(0, 0, 'E', [lineNode(2, 1, 'e1', []), blankNode(2), lineNode(2, 3, 'e2', [])]), + lineNode(0, 4, 'F', [blankNode(5), lineNode(2, 6, 'f1', [])]), + lineNode(0, 7, 'G', [lineNode(2, 8, 'g1', [])]), + blankNode(9), + lineNode(0, 10, 'H', []), + blankNode(11), + ]) + ); + }); + + test('combineBraces', function () { + const tree = parseTree(dedent` +A { +} +B + b1 { + bb1 + } + b2 { + bb2 + + } +} +C { + c1 + c2 + c3 + c4 +} +`); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'A {', [lineNode(0, 1, '}', [], 'closer')]), + lineNode(0, 2, 'B', [ + lineNode(2, 3, 'b1 {', [lineNode(4, 4, 'bb1', []), lineNode(2, 5, '}', [], 'closer')]), + lineNode(2, 6, 'b2 {', [ + lineNode(4, 7, 'bb2', []), + blankNode(8), + lineNode(2, 9, '}', [], 'closer'), + ]), + lineNode(0, 10, '}', [], 'closer'), + ]), + lineNode(0, 11, 'C {', [ + lineNode(4, 12, 'c1', []), + lineNode(4, 13, 'c2', []), + lineNode(2, 14, 'c3', []), + lineNode(2, 15, 'c4', []), + lineNode(0, 16, '}', [], 'closer'), + ]), + ]) + ); + // Running the optimisation twice doesn't change the result + let newTree = <IndentationTree<string>>JSON.parse(JSON.stringify(tree)); + newTree = combineClosersAndOpeners(newTree); + compareTreeWithSpec(newTree, tree); + }); +}); + +/** + * Many examples in this suite are taken from + * https://docs.google.com/document/d/1WxjTDzx8Qbf4Bklrp9KwiQsB4-kTOloAR5h86np3_OM/edit# + */ +suite('Test bracket indentation spec', function () { + test('Opener merged to older sibling', function () { + const source = dedent` +A +( + B + C`; + const treeRaw = parseRaw(source); + const treeCode = parseTree(source, ''); + + // the raw indentation indicates line 1 is the parent of the following lines + compareTreeWithSpec( + treeRaw, + topNode([lineNode(0, 0, 'A', []), lineNode(0, 1, '(', [lineNode(4, 2, 'B', []), lineNode(4, 3, 'C', [])])]) + ); + + // the bracket parsing indicates line 0 is the parent + compareTreeWithSpec( + treeCode, + topNode([ + lineNode(0, 0, 'A', [ + lineNode(0, 1, '(', [], 'opener'), + lineNode(4, 2, 'B', []), + lineNode(4, 3, 'C', []), + ]), + ]) + ); + }); + + test('Closer merged, simplest case', function () { + const source = dedent` +A + B +)`; + const treeRaw = parseRaw(source); + const treeCode = parseTree(source, ''); + + // the raw indentation indicates line 2 is the sibling of 0 + compareTreeWithSpec( + treeRaw, + topNode([lineNode(0, 0, 'A', [lineNode(4, 1, 'B', [])]), lineNode(0, 2, ')', [])]) + ); + + // the bracket parsing indicates line 2 actually another child + compareTreeWithSpec( + treeCode, + topNode([lineNode(0, 0, 'A', [lineNode(4, 1, 'B', []), lineNode(0, 2, ')', [], 'closer')])]) + ); + }); + + test('Closer merged, multi-body case', function () { + const source = dedent` +A + B + C +) + ( + D + E +)`; + const treeRaw = parseRaw(source); + const treeCode = parseTree(source, ''); + + // before bracket parsing, A had two children, B and C + assert.strictEqual( + treeRaw.subs[0].subs.map(x => (x.type === 'line' ? x.sourceLine.trim() : 'v')).join(), + 'B,C' + ); + // after, it had three children, a virtual node, line node 3 and the closer 6 + assert.strictEqual( + treeCode.subs[0].subs.map(x => (x.type === 'line' ? x.sourceLine.trim() : 'v')).join(), + 'v,) + (,)' + ); + }); + + test('closer starting their next subblock, ifelse', function () { + const source = dedent` + if (new) { + print(โ€œhelloโ€) + print(โ€œworldโ€) + } else { + print(โ€œgoodbyeโ€) + }`; + const sourceParsedAsIf = dedent` + if (new) { + -> virtual + print(โ€œhelloโ€) + print(โ€œworldโ€) + } else { + print(โ€œgoodbyeโ€) + }`; + + const treeRaw = parseRaw(source); + const treeCode = parseTree(source, ''); + const treeExpected = parseAsIfVirtual(sourceParsedAsIf); + + compareTreeWithSpec( + treeRaw, + topNode([ + lineNode(0, 0, 'if (new) {', [ + lineNode(4, 1, 'print(โ€œhelloโ€)', []), + lineNode(4, 2, 'print(โ€œworldโ€)', []), + ]), + lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(โ€œgoodbyeโ€)', [])]), + lineNode(0, 5, '}', []), + ]) + ); + compareTreeWithSpec( + treeCode, + topNode([ + lineNode(0, 0, 'if (new) {', [ + virtualNode(0, [lineNode(4, 1, 'print(โ€œhelloโ€)', []), lineNode(4, 2, 'print(โ€œworldโ€)', [])]), + lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(โ€œgoodbyeโ€)', [])]), + lineNode(0, 5, '}', []), + ]), + ]) + ); + compareTreeWithSpec(treeCode, treeExpected, 'structure'); + }); +}); + +suite('Special indentation styles', function () { + test('Allman style example (function)', function () { + const source = dedent` + function test() + { + print(โ€œhelloโ€) + print(โ€œworldโ€) + }`; + + const treeRaw = parseRaw(source); + const treeCode = parseTree(source, ''); + + // the bracket parsing indicates line 0 is the parent + compareTreeWithSpec( + treeCode, + topNode([ + lineNode(0, 0, 'function test()', [ + lineNode(0, 1, '{', [], 'opener'), + lineNode(4, 2, 'print(โ€œhelloโ€)', []), + lineNode(4, 3, 'print(โ€œworldโ€)', []), + lineNode(0, 4, '}', [], 'closer'), + ]), + ]) + ); + + // the next line is also moved, but by the closing partof the spec, so not tested here + compareTreeWithSpec( + treeRaw, + topNode([ + lineNode(0, 0, 'function test()', []), + lineNode(0, 1, '{', [lineNode(4, 2, 'print(โ€œhelloโ€)', []), lineNode(4, 3, 'print(โ€œworldโ€)', [])]), + lineNode(0, 4, '}', []), + ]) + ); + }); + + /** This test is a case where our parsing isn't yet optimal */ + test('Allman style example (if-then-else)', function () { + const source = dedent` + if (condition) + { + print(โ€œhelloโ€) + print(โ€œworldโ€) + } + else + { + print(โ€œgoodbyeโ€) + print(โ€œphoneโ€) + } + `; + + const treeCode = parseTree(source, ''); + + // Currently, this is parsed the same as two consecutive if-statements, + // Because generic languages do not understand `else` should continue. + compareTreeWithSpec( + treeCode, + topNode([ + lineNode(0, 0, 'if (condition)', [ + lineNode(0, 1, '{', [], 'opener'), + lineNode(4, 2, 'print(โ€œhelloโ€)', []), + lineNode(4, 3, 'print(โ€œworldโ€)', []), + lineNode(0, 4, '}', [], 'closer'), + ]), + lineNode(0, 5, 'else ', [ + lineNode(0, 6, '{', [], 'opener'), + lineNode(4, 7, 'print(โ€œgoodbyeโ€)', []), + lineNode(4, 8, 'print(โ€œphoneโ€)', []), + lineNode(0, 9, '}', [], 'closer'), + ]), + ]) + ); + }); + + test('K&R style example (if-then-else)', function () { + const source = dedent` + if (condition) { + print(โ€œhelloโ€) + print(โ€œworldโ€) + } else { + print(โ€œgoodbyeโ€) + print(โ€œphoneโ€) + } + `; + + const treeCode = parseTree(source, ''); + + // Currently, this is parsed the same as two consecutive if-statements, + // Because generic languages do not understand `else` should continue. + compareTreeWithSpec( + treeCode, + topNode([ + lineNode(0, 0, 'if (condition) {', [ + virtualNode(0, [lineNode(4, 2, 'print(โ€œhelloโ€)', []), lineNode(4, 3, 'print(โ€œworldโ€)', [])]), + lineNode( + 0, + 4, + '} else {', + [lineNode(4, 5, 'print(โ€œgoodbyeโ€)', []), lineNode(4, 6, 'print(โ€œphoneโ€)', [])], + 'closer' + ), + lineNode(0, 7, '}', [], 'closer'), + ]), + ]) + ); + }); + + test('combineBraces GNU style indentation 1', function () { + let tree: IndentationTree<string> = parseRaw(dedent` +A + { + stmt + } +`); + labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); + tree = combineClosersAndOpeners(tree); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'A', [ + lineNode(2, 1, '{', [lineNode(4, 2, 'stmt', []), lineNode(2, 3, '}', [], 'closer')], 'opener'), + ]), + ]) + ); + }); + + test('combineBraces GNU style indentation 2', function () { + let tree: IndentationTree<string> = parseRaw(dedent` +B +{ + stmt + +} + + +end +`); + labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); + tree = combineClosersAndOpeners(tree); + tree = flattenVirtual(tree); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'B', [ + lineNode(0, 1, '{', [], 'opener'), + lineNode(4, 2, 'stmt', []), + blankNode(3), + lineNode(0, 4, '}', [], 'closer'), + ]), + blankNode(5), + blankNode(6), + lineNode(0, 7, 'end', []), + ]) + ); + }); + + test('combineBraces GNU style indentation 3', function () { + let tree: IndentationTree<string> = parseRaw(dedent` +C +{ + +} +`); + labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); + tree = combineClosersAndOpeners(tree); + tree = flattenVirtual(tree); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'C', [ + lineNode(0, 1, '{', [], 'opener'), + blankNode(2), + lineNode(0, 3, '}', [], 'closer'), + ]), + ]) + ); + }); + + test('combineBraces GNU style indentation 4', function () { + let tree: IndentationTree<string> = parseRaw(dedent` +D +{ + d + { + stmt + + } +} +`); + labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); + tree = combineClosersAndOpeners(tree); + tree = flattenVirtual(tree); + compareTreeWithSpec( + tree, + topNode([ + lineNode(0, 0, 'D', [ + lineNode(0, 1, '{', [], 'opener'), + lineNode(4, 2, 'd', [ + lineNode(4, 3, '{', [], 'opener'), + lineNode(8, 4, 'stmt', []), + blankNode(5), + lineNode(4, 6, '}', [], 'closer'), + ]), + lineNode(0, 7, '}', [], 'closer'), + ]), + ]) + ); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/languageMarker.test.ts b/completions-sample-code/vscode-node/prompt/src/test/languageMarker.test.ts new file mode 100644 index 0000000..34289b0 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/languageMarker.test.ts @@ -0,0 +1,178 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// we need useless escapes before `!` or some tooling breaks; contact @johanrosenkilde for details +import { + comment, + commentBlockAsSingles, + getLanguage, + getLanguageMarker, + hasLanguageMarker, + mdCodeBlockLangToLanguageId, +} from '../languageMarker'; +import { DocumentInfoWithOffset } from '../prompt'; +import * as assert from 'assert'; +import * as fs from 'fs'; +import { resolve } from 'path'; + +suite('LanguageMarker Test Suite', function () { + let doc: DocumentInfoWithOffset; + + setup(function () { + const source = fs.readFileSync(resolve(__dirname, 'testdata/example.py'), 'utf8'); + const languageId = 'python'; + + doc = { + uri: 'file:///home/user/test.py', + source, + languageId, + offset: 0, + }; + }); + + test('getLanguageMarker', function () { + doc.languageId = 'python'; + assert.strictEqual(getLanguageMarker(doc), '#!/usr/bin/env python3'); + doc.languageId = 'cpp'; + assert.strictEqual(getLanguageMarker(doc), 'Language: cpp'); + doc.languageId = 'css'; + assert.strictEqual(getLanguageMarker(doc), 'Language: css'); + doc.languageId = 'html'; + assert.strictEqual(getLanguageMarker(doc), '<!DOCTYPE html>'); + doc.languageId = 'php'; + assert.strictEqual(getLanguageMarker(doc), ''); + doc.languageId = 'yaml'; + assert.strictEqual(getLanguageMarker(doc), '# YAML data'); + doc.languageId = 'unknown'; + assert.strictEqual(getLanguageMarker(doc), 'Language: unknown'); + }); + + test('hasLanguageMarker', function () { + doc.languageId = 'python'; + doc.source = 'import mypants\ndef my_socks():\n pass'; + assert.ok(!hasLanguageMarker(doc)); + doc.source = '#!/bin/python\n' + doc.source; //Note: not the shebang we add ourselves + assert.ok(hasLanguageMarker(doc)); + + doc.languageId = 'html'; + doc.source = '<html><body><p>My favourite web page</p></body></html>'; + assert.ok(!hasLanguageMarker(doc)); + doc.source = '<!DOCTYPE html>' + doc.source; + assert.ok(hasLanguageMarker(doc)); + + doc.languageId = 'shellscript'; + doc.source = 'echo Wonderful script'; + assert.ok(!hasLanguageMarker(doc)); + doc.source = '#!/bin/bash\n' + doc.source; + assert.ok(hasLanguageMarker(doc)); + }); + + test('comment normal', function () { + assert.strictEqual(comment('', 'python'), '# '); + assert.strictEqual(comment('hello', 'python'), '# hello'); + assert.strictEqual(comment('hello', 'typescript'), '// hello'); + }); + + test('comment demonstrate multiple lines gives unintuitive result', function () { + assert.strictEqual(comment('hello\nworld', 'typescript'), '// hello\nworld'); + }); + + test('comment non-existing language', function () { + assert.strictEqual(comment('hello', 'nonexistent'), '// hello'); + }); + + test('comment normal with default', function () { + assert.strictEqual(comment('', 'python'), '# '); + assert.strictEqual(comment('', 'nonexistent'), '// '); + assert.strictEqual(comment('hello', 'nonexistent'), '// hello'); + }); + + test('commentBlockAsSingles normal', function () { + assert.strictEqual(commentBlockAsSingles('', 'python'), ''); + assert.strictEqual(commentBlockAsSingles('hello', 'python'), '# hello'); + assert.strictEqual(commentBlockAsSingles('hello\nworld', 'python'), '# hello\n# world'); + assert.strictEqual(commentBlockAsSingles('hello\nworld', 'typescript'), '// hello\n// world'); + }); + + test('commentBlockAsSingles trailing newline', function () { + assert.strictEqual(commentBlockAsSingles('hello\nworld\n', 'python'), '# hello\n# world\n'); + assert.strictEqual(commentBlockAsSingles('\n', 'python'), '# \n'); + }); + + test('commentBlockAsSingles nonexistent language', function () { + assert.strictEqual(commentBlockAsSingles('hello\nworld', 'nonexistent'), '// hello\n// world'); + }); + + test('commentBlockAsSingles with default', function () { + assert.strictEqual(commentBlockAsSingles('hello\nworld', 'python'), '# hello\n# world'); + assert.strictEqual(commentBlockAsSingles('hello\nworld', 'nonexistent'), '// hello\n// world'); + }); + + const markdownLanguageIdsTestCases = [ + { input: 'h', expected: 'c' }, + { input: 'py', expected: 'python' }, + { input: 'js', expected: 'javascript' }, + { input: 'ts', expected: 'typescript' }, + { input: 'cpp', expected: 'cpp' }, + { input: 'java', expected: 'java' }, + { input: 'cs', expected: 'csharp' }, + { input: 'rb', expected: 'ruby' }, + { input: 'php', expected: 'php' }, + { input: 'html', expected: 'html' }, + { input: 'css', expected: 'css' }, + { input: 'xml', expected: 'xml' }, + { input: 'sh', expected: 'shellscript' }, + { input: 'go', expected: 'go' }, + { input: 'rs', expected: 'rust' }, + { input: 'swift', expected: 'swift' }, + { input: 'kt', expected: 'kotlin' }, + { input: 'lua', expected: 'lua' }, + { input: 'sql', expected: 'sql' }, + { input: 'yaml', expected: 'yaml' }, + { input: 'md', expected: 'markdown' }, + { input: 'plaintext', expected: undefined }, + ]; + + markdownLanguageIdsTestCases.forEach(({ input, expected }) => { + test(`test markdownLanguageId ${input} to language id ${expected}`, function () { + const languageId = mdCodeBlockLangToLanguageId(input); + assert.strictEqual(languageId, expected); + }); + }); + + const getLanguageTestCases = [ + { input: 'python', expected: 'python', expCommentStart: '#', expCommentEnd: '' }, + { input: 'javascript', expected: 'javascript', expCommentStart: '//', expCommentEnd: '' }, + { input: 'typescript', expected: 'typescript', expCommentStart: '//', expCommentEnd: '' }, + { input: 'cpp', expected: 'cpp', expCommentStart: '//', expCommentEnd: '' }, + { input: 'java', expected: 'java', expCommentStart: '//', expCommentEnd: '' }, + { input: 'csharp', expected: 'csharp', expCommentStart: '//', expCommentEnd: '' }, + { input: 'ruby', expected: 'ruby', expCommentStart: '#', expCommentEnd: '' }, + { input: 'php', expected: 'php', expCommentStart: '//', expCommentEnd: '' }, + { input: 'html', expected: 'html', expCommentStart: '<!--', expCommentEnd: '-->' }, + { input: 'css', expected: 'css', expCommentStart: '/*', expCommentEnd: '*/' }, + { input: 'xml', expected: 'xml', expCommentStart: '<!--', expCommentEnd: '-->' }, + { input: 'shellscript', expected: 'shellscript', expCommentStart: '#', expCommentEnd: '' }, + { input: 'go', expected: 'go', expCommentStart: '//', expCommentEnd: '' }, + { input: 'rust', expected: 'rust', expCommentStart: '//', expCommentEnd: '' }, + { input: 'swift', expected: 'swift', expCommentStart: '//', expCommentEnd: '' }, + { input: 'kotlin', expected: 'kotlin', expCommentStart: '//', expCommentEnd: '' }, + { input: 'lua', expected: 'lua', expCommentStart: '--', expCommentEnd: '' }, + { input: 'sql', expected: 'sql', expCommentStart: '--', expCommentEnd: '' }, + { input: 'yaml', expected: 'yaml', expCommentStart: '#', expCommentEnd: '' }, + { input: 'markdown', expected: 'markdown', expCommentStart: '[]: #', expCommentEnd: '' }, + { input: 'plaintext', expected: 'plaintext', expCommentStart: '//', expCommentEnd: '' }, + { input: 'not-existed', expected: 'not-existed', expCommentStart: '//', expCommentEnd: '' }, + { input: undefined, expected: 'plaintext', expCommentStart: '//', expCommentEnd: '' }, + ]; + + getLanguageTestCases.forEach(({ input, expected, expCommentStart, expCommentEnd }) => { + test(`test getLanguage for language id ${input} to language id ${expected}`, function () { + const language = getLanguage(input); + assert.strictEqual(language.languageId, expected); + assert.strictEqual(language.lineComment.start, expCommentStart); + assert.strictEqual(language.lineComment.end, expCommentEnd); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/multisnippet.test.ts b/completions-sample-code/vscode-node/prompt/src/test/multisnippet.test.ts new file mode 100644 index 0000000..821ecfb --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/multisnippet.test.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { DocumentInfoWithOffset, SimilarFileInfo } from '../prompt'; +import { + SimilarFilesOptions, + defaultSimilarFilesOptions, + getSimilarSnippets, + nullSimilarFilesOptions, +} from '../snippetInclusion/similarFiles'; +import * as assert from 'assert'; +import dedent from 'ts-dedent'; + +suite('Test Multiple Snippet Selection', function () { + const docSource: string = dedent` + A + B + C + D| + E + F + G`; + const doc: DocumentInfoWithOffset = { + relativePath: 'source1', + uri: 'source1', + source: docSource, + languageId: 'python', + offset: docSource.indexOf('|'), // reference snippet will be A B C D + }; + + const similarFiles: SimilarFileInfo[] = [ + { + relativePath: 'similarFile1', + uri: 'similarFile1', + source: dedent` + A + B + C + H + X + Y + Z + `, + }, + { + relativePath: 'similarFile2', + uri: 'similarFile2', + source: dedent` + D + H + `, + }, + ]; + + const fixedWinDocSrc = + 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz' + .split('') + .join('\n'); + const fixedWinDoc: DocumentInfoWithOffset = { + relativePath: 'source1', + uri: 'source1', + source: fixedWinDocSrc, + languageId: 'python', + offset: fixedWinDocSrc.length, // Reference doc qrstuvqxyz with conservative option (10 characters), stuv...abc...xyz with eager (60 characters) + }; + + const fixedWinSimilarFiles: SimilarFileInfo[] = [ + { + relativePath: 'similarFile1', + uri: 'similarFile1', + source: 'abcdefghijklmno1234567890abcdefghijklmnopqrstuvwxyzabcdefghijklmno1234567890abcdefghijklmnopqrstuvwxyzabcdefghijklmno1234567890abcdefghijklmnopqrstuvwxyz' + .split('') + .join('\n'), + }, + ]; + + test('FixedWindow Matcher None', async function () { + /** Test under FixedWindow matcher no match gets picked up */ + const options: SimilarFilesOptions = nullSimilarFilesOptions; + const snippets = await getSimilarSnippets(doc, similarFiles, options); + + assert.deepStrictEqual(snippets, []); + }); + + test('FixedWindow Matcher Eager No Selection Option', async function () { + /** This is to test Multisnippet selection with FixedWindow Matcher and Eager Neibhbortab + * option. windows size for Eager option is 60 and minimum score threshold for inclusion is 0.0. + * We expect only 1 match from line 0 to 60. WIth no selection option, we expect the best match to be returned. + */ + const options: SimilarFilesOptions = defaultSimilarFilesOptions; + const snippetLocationsTop1 = (await getSimilarSnippets(fixedWinDoc, fixedWinSimilarFiles, options)).map( + snippet => [snippet.startLine, snippet.endLine] + ); + const correctSnippetLocations: number[][] = [[0, 60]]; + assert.deepStrictEqual(snippetLocationsTop1.sort(), correctSnippetLocations.sort()); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/parse.test.ts b/completions-sample-code/vscode-node/prompt/src/test/parse.test.ts new file mode 100644 index 0000000..4533f3d --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/parse.test.ts @@ -0,0 +1,30 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as parse from '../parse'; +import * as assert from 'assert'; +import Parser from 'web-tree-sitter'; + +suite('Tree-sitter Parsing Tests', function () { + test('language wasm loading', async function () { + await Parser.init(); + await parse.getLanguage('python'); + await parse.getLanguage('javascript'); + await parse.getLanguage('go'); + await parse.getLanguage('php'); + await parse.getLanguage('c'); + await parse.getLanguage('cpp'); + await assert.rejects(async () => await parse.getLanguage('xxx')); + }); + + suite('getBlockCloseToken', function () { + test('all', function () { + assert.strictEqual(parse.getBlockCloseToken('javascript'), '}'); + assert.strictEqual(parse.getBlockCloseToken('typescript'), '}'); + assert.strictEqual(parse.getBlockCloseToken('python'), null); + assert.strictEqual(parse.getBlockCloseToken('ruby'), 'end'); + assert.strictEqual(parse.getBlockCloseToken('go'), '}'); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/parseBlock.test.ts b/completions-sample-code/vscode-node/prompt/src/test/parseBlock.test.ts new file mode 100644 index 0000000..af5945b --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/parseBlock.test.ts @@ -0,0 +1,1667 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import dedent from 'ts-dedent'; + +import { getBlockParser } from '../parseBlock'; + +interface TestCase { + before: string; // text before the cursor + body?: string; // body of the block after the cursor + after?: string; // text after the block +} + +/** + * Trimming modes for IsEmptyBlockStartTestCase below. + */ +enum TrimMode { + NO_TRIM, + TRIM_TO_END_OF_LINE, + TRIM_TO_END_OF_INPUT, +} + +/** + * A convenience class for testing BlockParser.isEmptyBlockStart. + * + * To use this, pass a string containing a snippet of source code, and use + * ๐ŸŸข for cursor positions at which isEmptyBlockStart should return true, + * and โŒ for cursor positions where it should return false. Then call + * .test() to run the tests. + * + * By default, for each cursor position it trims the line from the cursor + * to the end (i.e., the cursor is always at the end of the line) before + * executing the test. Set the trimMode property to change this. + */ +class IsEmptyBlockStartTestCase { + private readonly text: string; + private readonly expectTrueOffsets: number[]; + private readonly expectFalseOffsets: number[]; + private trimMode = TrimMode.TRIM_TO_END_OF_INPUT; + + private constructor( + private readonly languageId: string, + testCase: string + ) { + let text = ''; + const expectTrueOffsets: number[] = []; + const expectFalseOffsets: number[] = []; + let i = 0; + // Must use for...of loop to avoid surrogate pair/UTF-16 weirdness + for (const char of testCase) { + switch (char) { + case '๐ŸŸข': + expectTrueOffsets.push(i); + break; + case 'โŒ': + expectFalseOffsets.push(i); + break; + default: + text += char; + i++; + break; + } + } + + if (expectTrueOffsets.length === 0 && expectFalseOffsets.length === 0) { + throw new Error('Test case must have at least one cursor'); + } + + this.text = text; + this.expectTrueOffsets = expectTrueOffsets; + this.expectFalseOffsets = expectFalseOffsets; + } + + private trimText(offset: number): string { + switch (this.trimMode) { + case TrimMode.NO_TRIM: + return this.text; + case TrimMode.TRIM_TO_END_OF_LINE: { + const nextNewline = this.text.indexOf('\n', offset); + const fromNewline = nextNewline >= 0 ? this.text.slice(nextNewline) : ''; + return this.text.slice(0, offset) + fromNewline; + } + case TrimMode.TRIM_TO_END_OF_INPUT: + return this.text.slice(0, offset); + } + } + + // TODO(eaftan): It would be nice if this could test arbitrary functions. + async test<T>(): Promise<void> { + const blockParser = getBlockParser(this.languageId); + for (const offset of this.expectTrueOffsets) { + const text = this.trimText(offset); + const msg = `${this.text.slice(0, offset)}โ–ˆ${this.text.slice(offset)}`; + // common helper to all breaks + assert.strictEqual(await blockParser.isEmptyBlockStart(text, offset), true, msg); + } + for (const offset of this.expectFalseOffsets) { + const text = this.trimText(offset); + const msg = `${this.text.slice(0, offset)}โ–ˆ${this.text.slice(offset)}`; + assert.strictEqual(await blockParser.isEmptyBlockStart(text, offset), false, msg); + } + } + + setTrimMode(mode: TrimMode): IsEmptyBlockStartTestCase { + this.trimMode = mode; + return this; + } + + static python(testCase: string): IsEmptyBlockStartTestCase { + return new IsEmptyBlockStartTestCase('python', testCase); + } + + static javascript(testCase: string): IsEmptyBlockStartTestCase { + return new IsEmptyBlockStartTestCase('javascript', testCase); + } + + static typescript(testCase: string): IsEmptyBlockStartTestCase { + return new IsEmptyBlockStartTestCase('typescript', testCase); + } + + static ruby(testCase: string): IsEmptyBlockStartTestCase { + return new IsEmptyBlockStartTestCase('ruby', testCase); + } + + static go(testCase: string): IsEmptyBlockStartTestCase { + return new IsEmptyBlockStartTestCase('go', testCase); + } +} + +function runTestCase(languageId: string, testCase: TestCase) { + const bodyWithAfter = (testCase.body || '') + (testCase.after || ''); + const text = testCase.before + bodyWithAfter; + const blockParser = getBlockParser(languageId); + + // block is expected to be empty if no body + const expectedEmpty = !testCase.body; + // block is expected to be finished after body, if there is a body and an after + const expectedFinish = testCase.body && testCase.after ? testCase.body.length : undefined; + + // cursor position is after the before text + const offset = testCase.before.length; + // print the text with a cursor indicator on failure + const prettyPrint = ('\n' + testCase.before + 'โ–ˆ' + bodyWithAfter).split('\n').join('\n\t| '); + + test(`empty block start:${expectedEmpty}`, async function () { + const isEmpty = await blockParser.isEmptyBlockStart(text, offset); + // test isEmpty matched expectation + assert.strictEqual(isEmpty, expectedEmpty, prettyPrint); + }); + + test(`block finish:${expectedFinish}`, async function () { + const isFinished = await blockParser.isBlockBodyFinished(testCase.before, bodyWithAfter, offset); + // test isFinished matched expectation + assert.strictEqual(isFinished, expectedFinish, prettyPrint); + }); +} + +function runTestCases(languageId: string, testCases: TestCase[]) { + for (const testCase of testCases) { + runTestCase(languageId, testCase); + } +} + +function getNodeStartTestCase(testCase: string): [string, number[], number[], number] { + let text = ''; + let i = 0; + let expectedResult = 0; + const positiveTests: number[] = []; + const rejectedTests: number[] = []; + + // Must use for...of loop to avoid surrogate pair/UTF-16 weirdness + for (const char of testCase) { + switch (char) { + //Test cases that should pass the test + case '๐ŸŸข': + positiveTests.push(i); + break; + //Test cases that should fail the test + case 'โŒ': + rejectedTests.push(i); + break; + //Location used for the assertions (begining of the node we want to detect) + case '๐Ÿ”ต': + expectedResult = i; + break; + default: + text += char; + i++; + break; + } + } + + return [text, positiveTests, rejectedTests, expectedResult]; +} + +/** + * Helper function for testing `getNodeStart` + * + * To use this, pass a language ID and a string containing a snippet of source code, and use + * ๐Ÿ”ต for a location that's used for assertion ( begining of the node we want to detect) + * ๐ŸŸข for cursor positions at which `getNodeStart` should return the position ๐Ÿ”ต, + * and โŒ for cursor positions where it shouldn't. + */ +async function testGetNodeStart(languageId: string, testCase: string) { + const blockParser = getBlockParser(languageId); + const [code, positiveOffsets, rejectedOffsets, expected_result] = getNodeStartTestCase(testCase); + for (const offset of positiveOffsets) { + const start = await blockParser.getNodeStart(code, offset); + assert.strictEqual(start, expected_result, 'Should get beginning of the scope'); + } + for (const offset of rejectedOffsets) { + const start = await blockParser.getNodeStart(code, offset); + assert.notStrictEqual( + start, + expected_result, + `Should not get begining of the scope - tested offset: ${offset}` + ); + } +} + +suite('parseBlock Tests', function () { + suite('getBlockParser tests', function () { + test('Supported and unsupported languages', function () { + const supportedLanguages = ['python', 'javascript', 'typescript', 'go', 'ruby']; + for (const language of supportedLanguages) { + assert.ok(getBlockParser(language)); + } + + // Taken from https://insights.stackoverflow.com/survey/2020#most-popular-technologies and + // https://code.visualstudio.com/docs/languages/identifiers + const unsupportedLanguages = ['sql', 'java', 'shellscript', 'php', 'cpp', 'c', 'kotlin']; + for (const language of unsupportedLanguages) { + assert.throws(() => getBlockParser(language)); + } + }); + }); + + suite('Python isEmptyBlockStart tests', function () { + test('Invalid positions', async function () { + const text = dedent` + def foo(): + pass + `; + const blockParser = getBlockParser('python'); + await assert.rejects(blockParser.isEmptyBlockStart(text, text.length + 1)); + }); + + test('simple examples', async function () { + const testCases: IsEmptyBlockStartTestCase[] = [ + IsEmptyBlockStartTestCase.python(dedent` + โŒdโŒeโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + โŒeโŒlโŒiโŒf๐ŸŸข ๐ŸŸขb๐ŸŸขa๐ŸŸขr๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + eโŒlโŒsโŒe๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒssโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + eโŒlโŒsโŒe๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒtโŒrโŒy๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + โŒeโŒxโŒcโŒeโŒpโŒt๐ŸŸข ๐ŸŸขE๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + โŒfโŒiโŒnโŒaโŒlโŒlโŒy๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒtโŒrโŒy๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + โŒfโŒiโŒnโŒaโŒlโŒlโŒy๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸข ๐ŸŸขb๐ŸŸขa๐ŸŸขr๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒwโŒhโŒiโŒlโŒe๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒwโŒiโŒtโŒh๐ŸŸข ๐ŸŸขo๐ŸŸขp๐ŸŸขe๐ŸŸขn๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸขa๐ŸŸขs๐ŸŸข ๐ŸŸขf๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + โŒcโŒlโŒaโŒsโŒs๐ŸŸข ๐ŸŸขF๐ŸŸขo๐ŸŸขo๐ŸŸข:๐ŸŸข + ๐ŸŸขpโŒaโŒsโŒsโŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('func_decl', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.python('โŒdโŒeโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข:๐ŸŸข'), + IsEmptyBlockStartTestCase.python(dedent` + โŒdโŒeโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข:๐ŸŸข + ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข + ๐ŸŸข + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('multiline_func_decl', async function () { + const testCase = IsEmptyBlockStartTestCase.python(dedent` + โŒdโŒeโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸขa๐ŸŸข,๐ŸŸข + ๐ŸŸขb๐ŸŸข,๐ŸŸข + ๐ŸŸขc๐ŸŸข)๐ŸŸข:๐ŸŸข + ๐ŸŸข + `); + + await testCase.test(); + }); + + test('func_decl_in_middle_of_file', async function () { + // Trailing whitespace is intentional, do not remove! + const testCase = IsEmptyBlockStartTestCase.python( + dedent` + """This is a module.""" + import foo + + โŒdโŒeโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขu๐ŸŸขn๐ŸŸขc๐ŸŸข1๐ŸŸข(๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸข ๐ŸŸข + + print("Running at toplevel") + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE); + // break 1 + await testCase.test(); + }); + + test('func_decl_with_type_hints', async function () { + const testCase = IsEmptyBlockStartTestCase.python( + 'โŒdโŒeโŒf๐ŸŸข ๐ŸŸขs๐ŸŸขu๐ŸŸขm๐ŸŸข(๐ŸŸขa๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข,๐ŸŸข ๐ŸŸขb๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข ๐ŸŸข-๐ŸŸข>๐ŸŸข ๐ŸŸขI๐ŸŸขn๐ŸŸขt๐ŸŸข:๐ŸŸข' + ); + await testCase.test(); + }); + + test('block not empty', async function () { + const testCase = IsEmptyBlockStartTestCase.python( + dedent` + def func1(): + โŒ + passโŒ + โŒ + ` + ).setTrimMode(TrimMode.NO_TRIM); + await testCase.test(); + }); + + test('docstring', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.python(dedent` + def my_func(): + ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข"๐ŸŸข"๐ŸŸข"๐ŸŸขT๐ŸŸขh๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขa๐ŸŸข ๐ŸŸขd๐ŸŸขo๐ŸŸขc๐ŸŸขs๐ŸŸขt๐ŸŸขr๐ŸŸขi๐ŸŸขn๐ŸŸขg๐ŸŸข.๐ŸŸข"๐ŸŸข"๐ŸŸข"๐ŸŸข + `), + IsEmptyBlockStartTestCase.python(dedent` + def my_func(): + ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข'๐ŸŸข'๐ŸŸข'๐ŸŸขT๐ŸŸขh๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขa๐ŸŸข ๐ŸŸขd๐ŸŸขo๐ŸŸขc๐ŸŸขs๐ŸŸขt๐ŸŸขr๐ŸŸขi๐ŸŸขn๐ŸŸขg๐ŸŸข.๐ŸŸข'๐ŸŸข'๐ŸŸข'๐ŸŸข + `), + ]; + for (const testCase of testCases) { + // break 2 + await testCase.test(); + } + }); + + test('multiline docstring', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.python(dedent` + def my_func(): + """๐ŸŸขT๐ŸŸขh๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขa๐ŸŸข ๐ŸŸขm๐ŸŸขu๐ŸŸขl๐ŸŸขt๐ŸŸขi๐ŸŸขl๐ŸŸขi๐ŸŸขn๐ŸŸขe๐ŸŸข ๐ŸŸขd๐ŸŸขo๐ŸŸขc๐ŸŸขs๐ŸŸขt๐ŸŸขr๐ŸŸขi๐ŸŸขn๐ŸŸขg๐ŸŸข.๐ŸŸข + ๐ŸŸข + ๐ŸŸขH๐ŸŸขe๐ŸŸขr๐ŸŸขe๐ŸŸข'๐ŸŸขs๐ŸŸข ๐ŸŸขa๐ŸŸขn๐ŸŸขo๐ŸŸขt๐ŸŸขh๐ŸŸขe๐ŸŸขr๐ŸŸข ๐ŸŸขl๐ŸŸขi๐ŸŸขn๐ŸŸขe๐ŸŸข.๐ŸŸข"๐ŸŸข"๐ŸŸข"๐ŸŸข + `), + IsEmptyBlockStartTestCase.python(dedent` + def my_func(): + '''๐ŸŸขT๐ŸŸขh๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขi๐ŸŸขs๐ŸŸข ๐ŸŸขa๐ŸŸข ๐ŸŸขm๐ŸŸขu๐ŸŸขl๐ŸŸขt๐ŸŸขi๐ŸŸขl๐ŸŸขi๐ŸŸขn๐ŸŸขe๐ŸŸข ๐ŸŸขd๐ŸŸขo๐ŸŸขc๐ŸŸขs๐ŸŸขt๐ŸŸขr๐ŸŸขi๐ŸŸขn๐ŸŸขg๐ŸŸข.๐ŸŸข + ๐ŸŸข + ๐ŸŸขH๐ŸŸขe๐ŸŸขr๐ŸŸขe๐ŸŸข'๐ŸŸขs๐ŸŸข ๐ŸŸขa๐ŸŸขn๐ŸŸขo๐ŸŸขt๐ŸŸขh๐ŸŸขe๐ŸŸขr๐ŸŸข ๐ŸŸขl๐ŸŸขi๐ŸŸขn๐ŸŸขe๐ŸŸข.๐ŸŸข'๐ŸŸข'๐ŸŸข'๐ŸŸข + `), + ]; + + for (const testCase of testCases) { + // break 2 + await testCase.test(); + } + }); + + // TODO(eaftan): Ideally this test should pass, but the parse tree for unclosed docstrings + // is very odd, and I can't think of a way to distinuish between a broken parse tree without + // a block body and one with a block body. In practice in the extension, the check for + // isBlockBodyFinished prevents a multline suggestion from being given in this situation, + // because the block isn't finished until after the pass statement. + test.skip('docstring with body', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.python( + dedent` + def my_func():โŒ + "โŒ"โŒ"โŒTโŒhโŒiโŒsโŒ โŒiโŒsโŒ โŒaโŒ โŒdโŒoโŒcโŒsโŒtโŒrโŒiโŒnโŒgโŒ.โŒ"โŒ"โŒ"โŒ + pass + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.python( + dedent` + def my_func():โŒ + "โŒ"โŒ"โŒTโŒhโŒiโŒsโŒ โŒiโŒsโŒ โŒaโŒ โŒdโŒoโŒcโŒsโŒtโŒrโŒiโŒnโŒgโŒ.โŒ + + โŒHโŒeโŒrโŒeโŒ'โŒsโŒ โŒaโŒnโŒoโŒtโŒhโŒeโŒrโŒ โŒlโŒiโŒnโŒeโŒ.โŒ"โŒ"โŒ"โŒ + pass + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('Not EOL', async function () { + const testCase = IsEmptyBlockStartTestCase.python('def my_โŒfunc():').setTrimMode(TrimMode.NO_TRIM); + await testCase.test(); + }); + + test('if-elif-else', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.python(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข:๐ŸŸข + ๐ŸŸขpassโŒ + โŒeโŒlโŒiโŒf๐ŸŸข ๐ŸŸขb๐ŸŸขa๐ŸŸขr๐ŸŸข:๐ŸŸข + ๐ŸŸขpassโŒ + โŒeโŒlโŒsโŒe๐ŸŸข: + ๐ŸŸขpassโŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + // regression tests for #466 + test('block in error state', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.python(dedent` + def create_tables(conn):๐ŸŸข + """Create the tables students, courses and enrolled๐ŸŸข"""๐ŸŸข + conn = sqlite3.connect(results_db_path)โŒ + c = conn.cursor()โŒ + c.execute('''CREATE TABLE students (โŒ + โŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + if True:๐ŸŸข + conn = sqlite3.connect(results_db_path)โŒ + c = conn.cursor()โŒ + c.execute('''CREATE TABLE students (โŒ + โŒ + `), + IsEmptyBlockStartTestCase.python(dedent` + try:๐ŸŸข + conn = sqlite3.connect(results_db_path)โŒ + c = conn.cursor()โŒ + c.execute('''CREATE TABLE students (โŒ + โŒ + `), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + }); + + suite('JavaScript isEmptyBlockStart tests', function () { + test('arrow_function', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒ(โŒaโŒ)โŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒaโŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + // Note: We don't try to give a multline-suggestion immediately after "async". + // "async" is a keyword but not a reserved one, so it may be used as an + // identifier. Therefore when you have a partially written async function declaration, + // tree-sitter often parses it as a completed node of some other type (e.g. "async (a)" + // is parsed as a call of a function named "async" with arguments "a"). We'd have to do + // very hacky things to support this. + IsEmptyBlockStartTestCase.javascript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒ(โŒaโŒ)โŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒaโŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('try_statement, catch_clause, finally_clause', async function () { + const testCases: IsEmptyBlockStartTestCase[] = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒtโŒrโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒcโŒaโŒtโŒcโŒh๐ŸŸข ๐ŸŸข(๐ŸŸขe๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒtโŒrโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒfโŒiโŒnโŒaโŒlโŒlโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒtโŒrโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒcโŒaโŒtโŒcโŒh๐ŸŸข ๐ŸŸข(๐ŸŸขe๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒfโŒiโŒnโŒaโŒlโŒlโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('do_statement', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + โŒdโŒo๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒwโŒhโŒiโŒlโŒeโŒ โŒ(โŒtโŒrโŒuโŒeโŒ)โŒ;โŒ + `); + + await testCase.test(); + }); + + // tree-sitter's "for_in_statement" includes both for...in and for...of. + test('for_in_statement', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขv๐ŸŸขa๐ŸŸขr๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸข ๐ŸŸขo๐ŸŸขb๐ŸŸขj๐ŸŸขe๐ŸŸขc๐ŸŸขt๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขv๐ŸŸขa๐ŸŸขr๐ŸŸข ๐ŸŸขo๐ŸŸขf๐ŸŸข ๐ŸŸขo๐ŸŸขb๐ŸŸขj๐ŸŸขe๐ŸŸขc๐ŸŸขt๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('for_statement', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขi๐ŸŸข ๐ŸŸข=๐ŸŸข ๐ŸŸข0๐ŸŸข;๐ŸŸข ๐ŸŸขi๐ŸŸข ๐ŸŸข<๐ŸŸข ๐ŸŸข5๐ŸŸข;๐ŸŸข ๐ŸŸขi๐ŸŸข+๐ŸŸข+๐ŸŸข)๐ŸŸข {๐ŸŸข + ;โŒ + โŒ}โŒ + `); + + await testCase.test(); + }); + + test('if_statement', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒeโŒlโŒsโŒe๐ŸŸข ๐ŸŸขiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขb๐ŸŸขa๐ŸŸขr๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒeโŒlโŒsโŒe๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('method_definition', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + class Foo { + ๐ŸŸขbโŒaโŒrโŒ(โŒ)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + } + `); + + await testCase.test(); + }); + + test('switch_case, switch_default', async function () { + // We don't give multline suggestions for switch_case and switch_default + // because they are almost never blocks. + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + switch (foo) { + โŒcโŒaโŒsโŒeโŒ โŒbโŒaโŒrโŒ:โŒ + โŒbโŒrโŒeโŒaโŒkโŒ;โŒ + โŒdโŒeโŒfโŒaโŒuโŒlโŒtโŒ:โŒ + โŒbโŒrโŒeโŒaโŒkโŒ;โŒ + } + `); + + await testCase.test(); + }); + + test('while_statement', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + โŒwโŒhโŒiโŒlโŒe๐ŸŸข ๐ŸŸข(๐ŸŸขt๐ŸŸขr๐ŸŸขu๐ŸŸขe๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + test('with_statement', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + โŒwโŒiโŒtโŒh๐ŸŸข ๐ŸŸข(๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + // For the remaining node types (e.g. "function", "generator_function"), tree-sitter + // uses different node types to distinguish between ones used as declarations/statements + // and ones used as expressions. For example, "function_declaration" is a function declaration + // used as a declaration/statement, and "function" is the same thing used as an expression. + + test('function', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒlโŒeโŒtโŒ โŒfโŒ โŒ=โŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒlโŒeโŒtโŒ โŒfโŒ โŒ=โŒ โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('function_declaration', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข + ๐ŸŸข}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('generator_function', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒlโŒeโŒtโŒ โŒgโŒ โŒ=โŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒlโŒeโŒtโŒ โŒgโŒ โŒ=โŒ โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('generator_function_declaration', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.javascript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('class', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + โŒlโŒeโŒtโŒ โŒcโŒ โŒ=โŒ โŒcโŒlโŒaโŒsโŒs๐ŸŸข ๐ŸŸขC๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + test('class_declaration', async function () { + const testCase = IsEmptyBlockStartTestCase.javascript(dedent` + โŒcโŒlโŒaโŒsโŒs๐ŸŸข ๐ŸŸขC๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + // In JS/TS, when the code doesn't parse, it can be ambiguous whether + // two functions are siblings or one is a local function under the other + // (meaning the block is not empty and we should return false). + // + // TODO(eaftan): fix this and enable the test + test.skip('local or siblings', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข + function bar() {} + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.javascript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒnโŒ โŒfโŒoโŒoโŒ(โŒ)โŒ โŒ{โŒ + โŒ + function bar() {} + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.javascript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข + let a = 10; + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.javascript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒnโŒ โŒfโŒoโŒoโŒ(โŒ)โŒ โŒ{โŒ + โŒ + let a = 10; + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('regression test for #526', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.javascript( + dedent` + () => doIt(โŒ + โŒfโŒoโŒoโŒ.โŒfโŒoโŒoโŒ,โŒ + โŒbโŒaโŒrโŒ.โŒbโŒaโŒzโŒ,โŒ + โŒbโŒaโŒzโŒ.โŒbโŒaโŒzโŒ + ); + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.javascript( + dedent` + () => doIt(โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ + ); + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.javascript(dedent` + () => doIt(โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ + ); + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + }); + + suite('TypeScript isEmptyBlockStart tests', function () { + // "declare" is a contextual keyword, so we don't try to give a multiline + // suggestion until after "global," when it transitions from an identifer to a keyword. + test('ambient_declaration', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒdโŒeโŒcโŒlโŒaโŒrโŒeโŒ โŒgโŒlโŒoโŒbโŒaโŒl๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + + await testCase.test(); + }); + + // "namespace" is a contextual keyword, so we don't try to give a multiline + // suggestion until the open quote, when it transitions from an identifer to a keyword. + test('internal_module', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒnโŒaโŒmโŒeโŒsโŒpโŒaโŒcโŒeโŒ โŒ"๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข"๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + + await testCase.test(); + }); + + // "module" is a contextual keyword, so we don't try to give a multiline + // suggestion until the open quote, when it transitions from an identifer to a keyword. + test('module', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒmโŒoโŒdโŒuโŒlโŒeโŒ โŒ"๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข"๐ŸŸข ๐ŸŸข{๐ŸŸข + ;โŒ + โŒ}โŒ + `); + + await testCase.test(); + }); + + test('arrow_function', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒ(โŒaโŒ)โŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒ(โŒaโŒ:โŒ โŒsโŒtโŒrโŒiโŒnโŒgโŒ)โŒ:โŒ โŒvโŒoโŒiโŒdโŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒ(โŒaโŒ)โŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒ(โŒaโŒ:โŒ โŒsโŒtโŒrโŒiโŒnโŒgโŒ)โŒ:โŒ โŒvโŒoโŒiโŒdโŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒaโŒ โŒ=โŒ>๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + // TODO(eaftan): a catch variable may have a type annotation of "any" or "unknown", + // but the version of tree-sitter we're using doesn't support it yet. Add + // a test case when it's ready. See https://github.com/tree-sitter/tree-sitter-typescript/commit/cad2b85fd1136a5e12d3e089030b81d9fe4a0a08 + test('try_statement, catch_clause, finally_clause', async function () { + const testCases: IsEmptyBlockStartTestCase[] = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒtโŒrโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒcโŒaโŒtโŒcโŒh๐ŸŸข ๐ŸŸข(๐ŸŸขe๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒtโŒrโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒfโŒiโŒnโŒaโŒlโŒlโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒtโŒrโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒcโŒaโŒtโŒcโŒh๐ŸŸข ๐ŸŸข(๐ŸŸขe๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒfโŒiโŒnโŒaโŒlโŒlโŒy๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('do_statement', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒdโŒo๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒwโŒhโŒiโŒlโŒeโŒ โŒ(โŒtโŒrโŒuโŒeโŒ)โŒ;โŒ + `); + + await testCase.test(); + }); + + // tree-sitter's "for_in_statement" includes both for...in and for...of. + test('for_in_statement', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขv๐ŸŸขa๐ŸŸขr๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸข ๐ŸŸขo๐ŸŸขb๐ŸŸขj๐ŸŸขe๐ŸŸขc๐ŸŸขt๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขv๐ŸŸขa๐ŸŸขr๐ŸŸข ๐ŸŸขo๐ŸŸขf๐ŸŸข ๐ŸŸขo๐ŸŸขb๐ŸŸขj๐ŸŸขe๐ŸŸขc๐ŸŸขt๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('for_statement', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขi๐ŸŸข ๐ŸŸข=๐ŸŸข ๐ŸŸข0๐ŸŸข;๐ŸŸข ๐ŸŸขi๐ŸŸข ๐ŸŸข<๐ŸŸข ๐ŸŸข5๐ŸŸข;๐ŸŸข ๐ŸŸขi๐ŸŸข+๐ŸŸข+๐ŸŸข)๐ŸŸข {๐ŸŸข + ;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒoโŒr๐ŸŸข ๐ŸŸข(๐ŸŸขl๐ŸŸขe๐ŸŸขt๐ŸŸข ๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข ๐ŸŸข=๐ŸŸข ๐ŸŸข0๐ŸŸข;๐ŸŸข ๐ŸŸขi๐ŸŸข ๐ŸŸข<๐ŸŸข ๐ŸŸข5๐ŸŸข;๐ŸŸข ๐ŸŸขi๐ŸŸข+๐ŸŸข+๐ŸŸข)๐ŸŸข {๐ŸŸข + ;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('if_statement', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒeโŒlโŒsโŒe๐ŸŸข ๐ŸŸขiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขb๐ŸŸขa๐ŸŸขr๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒiโŒf๐ŸŸข ๐ŸŸข(๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ โŒeโŒlโŒsโŒe๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('method_definition', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + class Foo { + ๐ŸŸขbโŒaโŒrโŒ(โŒ)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + } + `), + IsEmptyBlockStartTestCase.typescript(dedent` + class Foo { + ๐ŸŸขbโŒaโŒrโŒ(โŒiโŒ:โŒ โŒiโŒnโŒtโŒ)๐ŸŸข:โŒ โŒv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + } + `), + // TODO(eaftan): fix sibling function issue and enable this test + // IsEmptyBlockStartTestCase.typescript(dedent` + // class Foo { + // fโŒoโŒoโŒ(โŒ)๐ŸŸข ๐ŸŸข{๐ŸŸข + // ๐ŸŸข}โŒ + + // โŒbโŒaโŒrโŒ(โŒ)๐ŸŸข ๐ŸŸข{๐ŸŸข + // ๐ŸŸข}โŒ + // } + // `).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('method_signature', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + class Foo { + ๐ŸŸขbโŒaโŒrโŒ(โŒ)๐ŸŸข;โŒ + } + `), + IsEmptyBlockStartTestCase.typescript(dedent` + class Foo { + ๐ŸŸขbโŒaโŒrโŒ(โŒiโŒ:โŒ โŒiโŒnโŒtโŒ)๐ŸŸข:โŒ โŒv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข;โŒ + } + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('switch_case, switch_default', async function () { + // We don't give multline suggestions for switch_case and switch_default + // because they are almost never blocks. + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + switch (foo) { + โŒcโŒaโŒsโŒeโŒ โŒbโŒaโŒrโŒ:โŒ + โŒbโŒrโŒeโŒaโŒkโŒ;โŒ + โŒdโŒeโŒfโŒaโŒuโŒlโŒtโŒ:โŒ + โŒbโŒrโŒeโŒaโŒkโŒ;โŒ + } + `); + + await testCase.test(); + }); + + test('while_statement', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒwโŒhโŒiโŒlโŒe๐ŸŸข ๐ŸŸข(๐ŸŸขt๐ŸŸขr๐ŸŸขu๐ŸŸขe๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + // For the remaining node types (e.g. "function", "generator_function"), tree-sitter + // uses different node types to distinguish between ones used as declarations/statements + // and ones used as expressions. For example, "function_declaration" is a function declaration + // used as a declaration/statement, and "function" is the same thing used as an expression. + + test('function', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒfโŒ โŒ=โŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒfโŒ โŒ=โŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒfโŒ โŒ=โŒ โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒfโŒ โŒ=โŒ โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸข(i๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('function_declaration', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(i๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข + ๐ŸŸข}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข ๐ŸŸข + ๐ŸŸข}โŒ + `), + IsEmptyBlockStartTestCase.typescript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(โŒxโŒ โŒ:โŒ โŒnโŒuโŒmโŒbโŒeโŒrโŒ,โŒ + ๐ŸŸขy๐ŸŸข ๐ŸŸข:๐ŸŸข ๐ŸŸขn๐ŸŸขu๐ŸŸขm๐ŸŸขb๐ŸŸขe๐ŸŸขr๐ŸŸข)๐ŸŸข ๐ŸŸข:๐ŸŸข ๐ŸŸขn๐ŸŸขu๐ŸŸขm๐ŸŸขb๐ŸŸขe๐ŸŸขr๐ŸŸข;โŒ + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸข + ๐ŸŸข + let x = 0; + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + function f(โŒ + /** first parameter */ + x: number, + /** second parameter */ + y: number); + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + function getPosition() : {โŒ + start: number,โŒ + end: numberโŒ + }; + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('generator_function', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒgโŒ โŒ=โŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒgโŒ โŒ=โŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒgโŒ โŒ=โŒ โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒgโŒ โŒ=โŒ โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('generator_function_declaration', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒsโŒyโŒnโŒcโŒ โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข*๐ŸŸข ๐ŸŸขg๐ŸŸขe๐ŸŸขn๐ŸŸขe๐ŸŸขr๐ŸŸขa๐ŸŸขt๐ŸŸขo๐ŸŸขr๐ŸŸข(๐ŸŸขi๐ŸŸข:๐ŸŸข ๐ŸŸขi๐ŸŸขn๐ŸŸขt๐ŸŸข)๐ŸŸข:๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('class', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒlโŒeโŒtโŒ โŒcโŒ โŒ=โŒ โŒcโŒlโŒaโŒsโŒs๐ŸŸข ๐ŸŸขC๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + test('class_declaration', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒcโŒlโŒaโŒsโŒs๐ŸŸข ๐ŸŸขC๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + test('abstract_class_declaration', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒaโŒbโŒsโŒtโŒrโŒaโŒcโŒtโŒ โŒcโŒlโŒaโŒsโŒs๐ŸŸข ๐ŸŸขC๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข;โŒ + โŒ}โŒ + `); + await testCase.test(); + }); + + // In JS/TS, when the code doesn't parse, it can be ambiguous whether + // two functions are siblings or one is a local function under the other + // (meaning the block is not empty and we should return false). + // + // TODO(eaftan): fix this and enable the test + test.skip('local or siblings', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข + function bar() {} + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒnโŒ โŒfโŒoโŒoโŒ(โŒ)โŒ โŒ{โŒ + โŒ + function bar() {} + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸขo๐ŸŸขo๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸข + let a = 10; + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒnโŒ โŒfโŒoโŒoโŒ(โŒ)โŒ โŒ{โŒ + โŒ + let a = 10; + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('regression test for #526', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.typescript( + dedent` + () => doIt(โŒ + โŒfโŒoโŒoโŒ.โŒfโŒoโŒoโŒ,โŒ + โŒbโŒaโŒrโŒ.โŒbโŒaโŒzโŒ,โŒ + โŒbโŒaโŒzโŒ.โŒbโŒaโŒzโŒ + ); + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript( + dedent` + () => doIt(โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ + ); + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.typescript(dedent` + () => doIt(โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ,โŒ + โŒ'โŒaโŒ'โŒ + ); + `), + ]; + + for (const testCase of testCases) { + await testCase.test(); + } + }); + + test('function type', async function () { + const testCase = IsEmptyBlockStartTestCase.typescript(dedent` + โŒfโŒuโŒnโŒcโŒtโŒiโŒoโŒn๐ŸŸข ๐ŸŸขf๐ŸŸข(๐ŸŸขc๐ŸŸขb๐ŸŸข:๐ŸŸข ๐ŸŸข(๐ŸŸข)๐ŸŸข ๐ŸŸข=๐ŸŸข>๐ŸŸข ๐ŸŸขv๐ŸŸขo๐ŸŸขi๐ŸŸขd๐ŸŸข)๐ŸŸข ๐ŸŸข{๐ŸŸข + ๐ŸŸขcโŒbโŒ(โŒ)โŒ;โŒ + โŒ}โŒ + `); + + await testCase.test(); + }); + }); + + suite('Ruby isEmptyBlockStart tests', function () { + test('simple examples', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.ruby(dedent` + def ๐ŸŸขgreet๐ŸŸข + ๐ŸŸขputs "Hello"โŒ + โŒputs "Bye"โŒ + end + `), + IsEmptyBlockStartTestCase.ruby( + dedent` + def ๐ŸŸขgreetโŒ + ๐ŸŸขputs "Hello"โŒ + end + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.ruby( + dedent` + def ๐ŸŸขgreetโŒ + โŒputs "Hello"โŒ + โŒputs "Bye"โŒ + end + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + }); + + suite('Go isEmptyBlockStart tests', function () { + test('simple examples', async function () { + const testCases = [ + IsEmptyBlockStartTestCase.go(dedent` + func ๐ŸŸขgreet๐ŸŸข()๐ŸŸข {๐ŸŸข + ๐ŸŸขfmt.Println("Hello")โŒ + โŒfmt.Println("Bye")โŒ + } + `), + IsEmptyBlockStartTestCase.go( + dedent` + func ๐ŸŸขgreet๐ŸŸข()๐ŸŸข {โŒ + ๐ŸŸขfmt.Println("Hello")โŒ + } + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + IsEmptyBlockStartTestCase.go( + dedent` + func ๐ŸŸขgreet๐ŸŸข()๐ŸŸข {โŒ + โŒfmt.Println("Hello")โŒ + โŒfmt.Println("Bye")โŒ + } + ` + ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), + ]; + for (const testCase of testCases) { + await testCase.test(); + } + }); + }); + + suite('python block body tests', function () { + const pythonBlockTests: TestCase[] = [ + { before: 'def foo():', body: '\n\tpass' }, + { before: 'def foo', body: '():\n\tpass', after: '\npass' }, + { before: 'def foo():', body: '\n\tpass', after: '\npass' }, + { before: 'def foo():', body: '\n\tpass', after: '\n\t\npass' }, + { before: 'def foo(arg1', body: '):\n\tpass', after: '\npass' }, + { before: 'def foo(arg1', body: '\n\t\t):\n\tpass', after: '\npass' }, + { before: 'def foo(arg1,', body: ' arg2):\n\tpass', after: '\npass' }, + { before: 'def foo', body: '():\n\tpass', after: '\n\npass' }, + { before: 'def foo' }, + { before: 'def foo', body: '():\n\t1+1\n\t# comment' }, + { before: 'def foo', body: '():\n\t1+1\n\t# comment1', after: '\n# comment2' }, + { before: 'def foo', body: '():\n\t# comment' }, + { before: 'def foo', body: '():\n\t1+1 # comment1', after: '\n# comment2' }, + { before: 'def foo', body: '():\n\t# comment1\n\t1+1', after: '\n# comment2' }, + { before: 'def foo', body: '():\n\t# comment1\n\t# comment2' }, + { before: 'def foo', body: '():\n\t# comment1\n\t# comment2', after: '\n# comment3' }, + { before: 'def foo', body: '(): #comment1' }, + { before: 'def foo', body: '():#comment1' }, + { before: 'try:', after: '\nexcept: pass' }, + { before: 'try:', body: '\n\t1+1', after: '\nexcept: pass' }, + { before: 'try:\n\tpass\nfinally:\n\tif 1:', body: '\n\t\tpass', after: '\npass' }, + { before: 'try:\n\tpass\nfinally:\n\tif 1:', after: '\npass' }, + { before: 'if 1:\n\tpass\nelse:\n\tif 2:', after: '\npass' }, + { before: 'if 1:\n\tpass\nelse:\n\tif 2:', after: '\n\tpass' }, + { before: 'if 1:\n\tpass\nelse:\n\tif 2:', after: '\n\n\tpass' }, + { + before: 'class C:\n\t"""docstring"""\n', + body: '\tdef foo():\n\t\tpass\n\tdef bar():\n\t\tpass', + after: '\npass', + }, + { before: 'class C:\n', body: '\tdef foo():\n\tpass\n\tdef bar():\n\t\tpass', after: '\npass' }, + { + before: 'for ', + body: ` record in records:\n\taccount_id = record'actor_id']\n\trecord['account_tier'] = account_tiers[account_id]`, + after: '\n\nprint(records)', + }, + ]; + runTestCases('python', pythonBlockTests); + }); + + suite('Python getBlockStart tests', function () { + test('class_definition', async function () { + const code = dedent` + ๐Ÿ”ตclass MyClass:๐ŸŸข + ๐ŸŸข"""A simple๐ŸŸข example class"""๐ŸŸข + ๐ŸŸขi = 12๐ŸŸข345๐ŸŸข + ๐ŸŸข + โŒdefโŒ f(self):โŒ + โŒreturnโŒ 'helloโŒ world'โŒ + + `; + + await testGetNodeStart('python', code); + }); + + test('elif_clause', async function () { + const code = dedent` + def โŒsample():โŒ + โŒif 1โŒ: + โŒpassโŒ + ๐Ÿ”ตelif๐ŸŸข 2๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + โŒelse:โŒ + โŒpassโŒ + `; + + await testGetNodeStart('python', code); + }); + + test('else_clause', async function () { + const code = dedent` + โŒdef โŒsample():โŒ + โŒif 1:โŒ + โŒpassโŒ + โŒelif 2:โŒ + โŒpassโŒ + ๐Ÿ”ตelse๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + `; + + await testGetNodeStart('python', code); + }); + + test('except_clause', async function () { + const code = dedent` + โŒdefโŒ โŒsampleโŒ()โŒ:โŒ + โŒtry:โŒ + โŒpassโŒ + ๐Ÿ”ตexcept๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + `; + + await testGetNodeStart('python', code); + }); + + test('finally_clause', async function () { + const code = dedent` + โŒdefโŒ saโŒmpleโŒ()โŒ:โŒ + โŒtry: + โŒpassโŒ + ๐Ÿ”ตfinally๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + `; + + await testGetNodeStart('python', code); + }); + + test('for_statement', async function () { + const code = dedent` + โŒdefโŒ โŒsample(โŒ):โŒ + โŒfruitsโŒ = โŒ["apple", "banana", "cherry"]โŒ + ๐Ÿ”ตfor๐ŸŸข x in๐ŸŸข fr๐ŸŸขuits๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + `; + + await testGetNodeStart('python', code); + }); + + test('function_definition', async function () { + const code = dedent` + ๐Ÿ”ตdef๐ŸŸข sam๐ŸŸขple๐ŸŸข(๐ŸŸข)๐ŸŸข: + ๐ŸŸข"""Sample ๐ŸŸขcomment"""๐ŸŸข + ๐ŸŸขfruits๐ŸŸข = ๐ŸŸข["apple", ๐ŸŸข"banana",๐ŸŸข "cherry"]๐ŸŸข + โŒforโŒ xโŒ inโŒ fruitsโŒ:โŒ + โŒpโŒaโŒsโŒsโŒ + `; + + await testGetNodeStart('python', code); + }); + + test('if_statement', async function () { + const code = dedent` + โŒdef โŒsampleโŒ(โŒ)โŒ:โŒ + ๐Ÿ”ตif ๐ŸŸข1๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + โŒelifโŒ 2:โŒ + โŒpass + โŒelse:โŒ + โŒpass + `; + + await testGetNodeStart('python', code); + }); + + test('try_statement', async function () { + const code = dedent` + โŒdefโŒ โŒsampleโŒ(โŒ)โŒ:โŒ + ๐Ÿ”ตtry๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + โŒfinโŒallโŒy:โŒ + โŒpassโŒ + `; + + await testGetNodeStart('python', code); + }); + + test('while_statement', async function () { + const code = dedent` + โŒdefโŒ saโŒmple(โŒ)โŒ:โŒ + ๐Ÿ”ตwhile๐ŸŸข ๐ŸŸขTr๐ŸŸขue๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + `; + + await testGetNodeStart('python', code); + }); + + test('with_statement', async function () { + const code = dedent` + โŒdefโŒ โŒsaโŒmpleโŒ(โŒ)โŒ:โŒ + ๐Ÿ”ตwith๐ŸŸข ๐ŸŸขopen๐ŸŸข(๐ŸŸข'fil๐ŸŸขe_pa๐ŸŸขth'๐ŸŸข, ๐ŸŸข'w')๐ŸŸข ๐ŸŸขas๐ŸŸข ๐ŸŸขf๐ŸŸขi๐ŸŸขl๐ŸŸขe๐ŸŸข:๐ŸŸข + ๐ŸŸขp๐ŸŸขa๐ŸŸขs๐ŸŸขs + `; + + await testGetNodeStart('python', code); + }); + }); + + // tests for JavaScript and TypeScript: `โฆƒ...โฆ„` delineates the body, `ใ€š...ใ€›` the type annotations, + // which are stripped off for JavaScript + + const test1 = dedent` + function getTextOrNull(documentใ€š: doc | nullใ€›) { + if (document === null) + โฆƒ return null; + return document.getText(); + }โฆ„ + + // this is a comment`; + + const test2 = dedent` + function getB(capitalใ€š: booleanใ€›) { + if (capital) { + return "B"; + } else {โฆƒ + return "b"; + }โฆ„ + }`; + + function mkTestCase(src: string, stripTypes: boolean) { + if (stripTypes) { src = src.replace(/ใ€š.*?ใ€›/g, ''); } + const bodyStart = src.indexOf('โฆƒ'); + const bodyEnd = src.indexOf('โฆ„'); + return { + before: src.slice(0, bodyStart), + body: src.slice(bodyStart + 1, bodyEnd), + after: src.slice(bodyEnd + 1), + }; + } + + suite('JavaScript isBlockBodyFinished tests', function () { + runTestCases('javascript', [mkTestCase(test1, true), mkTestCase(test2, true)]); + }); + + suite('TypeScript isBlockBodyFinished tests', function () { + runTestCases('typescript', [mkTestCase(test1, false), mkTestCase(test2, false)]); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/similarFiles.test.ts b/completions-sample-code/vscode-node/prompt/src/test/similarFiles.test.ts new file mode 100644 index 0000000..fb6b652 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/similarFiles.test.ts @@ -0,0 +1,478 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import dedent from 'ts-dedent'; +import { DocumentInfoWithOffset, SimilarFileInfo } from '../prompt'; +import { FixedWindowSizeJaccardMatcher, computeScore } from '../snippetInclusion/jaccardMatching'; +import { ScoredSnippetMarker, SortOptions, splitIntoWords } from '../snippetInclusion/selectRelevance'; +import { + SimilarFilesOptions, + conservativeFilesOptions, + defaultCppSimilarFilesOptions, + defaultSimilarFilesOptions, + getSimilarSnippets, + nullSimilarFilesOptions, +} from '../snippetInclusion/similarFiles'; +import { SnippetWithProviderInfo } from '../snippetInclusion/snippets'; +import { initializeTokenizers } from '../tokenization'; + +async function retrieveAllSnippetsWithJaccardScore( + objectDoc: SimilarFileInfo, + referenceDoc: SimilarFileInfo, + windowLength: number, + sortOption: SortOptions +): Promise<ScoredSnippetMarker[]> { + const referenceDocWithOffset: DocumentInfoWithOffset = { + ...referenceDoc, + languageId: '', + offset: referenceDoc.source.length, + }; + const matcher = FixedWindowSizeJaccardMatcher.FACTORY(windowLength).to(referenceDocWithOffset); + const match = await matcher.retrieveAllSnippets(objectDoc, sortOption); + + return match; +} + +async function findBestJaccardMatch( + objectDoc: SimilarFileInfo, + referenceDoc: SimilarFileInfo, + windowLength: number +): Promise<SnippetWithProviderInfo[]> { + const referenceDocWithOffset: DocumentInfoWithOffset = { + ...referenceDoc, + languageId: '', + offset: referenceDoc.source.length, + }; + const matcher = FixedWindowSizeJaccardMatcher.FACTORY(windowLength).to(referenceDocWithOffset); + const match = await matcher.findBestMatch(objectDoc, defaultCppSimilarFilesOptions.maxSnippetsPerFile); + return match; +} + +suite('selectRelevance Test Suite', function () { + setup(async function () { + await initializeTokenizers; + }); + + test('findBestJaccardMatch computes correct score of two single lines', async function () { + // 100% match if equal + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good morning', uri: 'file:///home/user/test.js' }, + { source: 'good morning', uri: 'file:///home/user/test.js' }, + 1 + ) + )[0].score, + 1 + ); + // no match if different + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good morning', uri: 'file:///home/user/test.js' }, + { source: 'bad night', uri: 'file:///home/user/test.js' }, + 1 + ) + ).length, + 0 + ); + // 33% match if 1 same, 1 different (because it's 1 overlap of 3 tokens in total) + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good morning', uri: 'file:///home/user/test.js' }, + { source: 'good night', uri: 'file:///home/user/test.js' }, + 1 + ) + )[0].score, + 1 / 3 + ); + // 50% match if half the tokens are missing (because it's 1 overlap of 2 tokens in total) + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good morning', uri: 'file:///home/user/test.js' }, + { source: 'good', uri: 'file:///home/user/test.js' }, + 1 + ) + )[0].score, + 0.5 + ); + // order is ignored + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good morning', uri: 'file:///home/user/test.js' }, + { source: 'morning good', uri: 'file:///home/user/test.js' }, + 1 + ) + )[0].score, + 1 + ); + // so are stop words + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good morning', uri: 'file:///home/user/test.js' }, + { source: 'morning is good', uri: 'file:///home/user/test.js' }, + 1 + ) + )[0].score, + 1 + ); + // and non alphanumeric_ characters + assert.strictEqual( + ( + await findBestJaccardMatch( + { source: 'good !morning sunshine', uri: 'file:///home/user/test.js' }, + { source: 'goodรขโ€šยฌmorning,sunshine', uri: 'file:///home/user/test.js' }, + 1 + ) + )[0].score, + 1 + ); + }); + + /** + * When requesting matches with a certain length, + * the returns have that length + */ + test('findBestJaccardMatch respects windowLength', async function () { + // no window no match + assert.strictEqual( + ( + await findBestJaccardMatch( + { + source: 'good morning\ngood night\nthe day\nis bright', + uri: 'file:///home/user/test.js', + }, + { + source: 'good morning\ngood night\nthe day\nis bright', + uri: 'file:///home/user/test.js', + }, + 0 + ) + ).length, + 0 + ); + // for identical object and reference docs + for (const n of [1, 2]) { + assert.strictEqual( + ( + await findBestJaccardMatch( + { + source: 'good morning\ngood night\nthe day\nis bright', + uri: 'file:///home/user/test.js', + }, + { + source: 'good morning\ngood night\nthe day\nis bright', + uri: 'file:///home/user/test.js', + }, + n + ) + )[0].snippet.split('\n').length, + n + ); + } + // if the ref doc is shorter + for (const n of [1, 2]) { + assert.strictEqual( + ( + await findBestJaccardMatch( + { + source: 'good morning\ngood night\nthe day\nis bright', + uri: 'file:///home/user/test.js', + }, + { source: 'good night', uri: 'file:///home/user/test.js' }, + n + ) + )[0].snippet.split('\n').length, + n + ); + } + // if the ref doc is longer + for (const n of [1, 2]) { + const matches = await findBestJaccardMatch( + { + source: 'good morning\ngood night\nthe day\nis bright', + uri: 'file:///home/user/test.js', + }, + { + source: 'good morning\ngood night\nthe day\nis bright\nthe sun', + uri: 'file:///home/user/test.js', + }, + n + ); + if (n === 1) { assert.strictEqual(matches.length, 0); } + else if (n === 2) { + assert.strictEqual(matches.length, 1); + assert.strictEqual(matches[0].snippet.split('\n').length, n > 1 ? n : []); + } else { + throw new Error('Unexpected value for `n`'); + } + } + }); + + test('findBestJaccardMatch returns the best match', async function () { + assert.strictEqual( + ( + await findBestJaccardMatch( + { + source: ['abcd', 'efgh', 'ijkl', 'mnop', 'qrst', 'uvwx', 'yz'].join('\n'), + uri: 'file:///home/user/test.js', + }, + { source: ['ijkl', 'qrst'].join('\n'), uri: 'file:///home/user/test.js' }, + 3 + ) + )[0].snippet, + ['ijkl', 'mnop', 'qrst'].join('\n') + ); + }); + + test('findBestJaccardMatch works on strings with or without a newline at the end', async function () { + assert.strictEqual( + ( + await findBestJaccardMatch( + { + source: ['abcd', 'efgh', 'ijkl', 'mnop', 'qrst', 'uvwx', 'yz'].join('\n'), + uri: 'file:///home/user/test.js', + }, + { source: ['ijkl', 'qrst'].join('\n'), uri: 'file:///home/user/test.js' }, + 3 + ) + )[0].snippet, + ['ijkl', 'mnop', 'qrst'].join('\n') + ); + }); + + test('Tokenization splits words on whitespace', function () { + assert.deepStrictEqual(splitIntoWords('def hello'), ['def', 'hello']); + assert.deepStrictEqual(splitIntoWords('def hello'), ['def', 'hello']); + assert.deepStrictEqual(splitIntoWords('def \n\t hello'), ['def', 'hello']); + }); + + test('Tokenization keeps numbers attached to words', function () { + assert.deepStrictEqual(splitIntoWords('def hello1:\n\treturn world49'), ['def', 'hello1', 'return', 'world49']); + }); + + test('Tokenization splits words on special characters', function () { + assert.deepStrictEqual(splitIntoWords('def hello(world):\n\treturn a.b+1'), [ + 'def', + 'hello', + 'world', + 'return', + 'a', + 'b', + '1', + ]); + }); + + test('Tokenization splits words on underscores', function () { + assert.deepStrictEqual(splitIntoWords(`def hello_world:\n\treturn 'I_am_a_sentence!'`), [ + 'def', + 'hello', + 'world', + 'return', + 'I', + 'am', + 'a', + 'sentence', + ]); + }); + + test('Find all snippets.', async function () { + const windowLength = 2; + const doc1 = { + source: 'or not\ngood morning\ngood night\nthe day\nis bright\nthe morning sun\nis hot', + uri: 'file:///home/user/test.js', + }; + const refDoc = { + source: 'good morning good night the day is bright', + languageId: '', + uri: 'file:///home/user/test.js', + }; + assert.deepStrictEqual( + await retrieveAllSnippetsWithJaccardScore(doc1, refDoc, windowLength, SortOptions.None), + [ + { score: 0.6, startLine: 1, endLine: 3 }, + { score: 0.4, startLine: 3, endLine: 5 }, + { score: 0.14285714285714285, startLine: 5, endLine: 7 }, + ] + ); + + assert.deepStrictEqual( + await retrieveAllSnippetsWithJaccardScore(doc1, refDoc, windowLength, SortOptions.Ascending), + [ + { score: 0.14285714285714285, startLine: 5, endLine: 7 }, + { score: 0.4, startLine: 3, endLine: 5 }, + { score: 0.6, startLine: 1, endLine: 3 }, + ] + ); + + assert.deepStrictEqual( + await retrieveAllSnippetsWithJaccardScore(doc1, refDoc, windowLength, SortOptions.Descending), + [ + { score: 0.6, startLine: 1, endLine: 3 }, + { score: 0.4, startLine: 3, endLine: 5 }, + { score: 0.14285714285714285, startLine: 5, endLine: 7 }, + ] + ); + }); + + test('Test Jaccard similarity.', function () { + const bagOfWords1 = 'one two three four five'; + const bagOfWords2 = 'zone ztwo zthree zfour zfive'; + const bagOfWords3 = 'one two three four five six'; // single word difference with bagOfWords1 + const bagOfWords4 = 'one ztwo zthree zfour zfive'; // single word intersection with bagOfWords1 + const bagOfWords5 = 'one ztwo ztwo zthree zfour zfive'; // repeated words + assert.strictEqual(computeScore(new Set(splitIntoWords(bagOfWords1)), new Set(splitIntoWords(bagOfWords2))), 0); + assert.strictEqual(computeScore(new Set(splitIntoWords(bagOfWords1)), new Set(splitIntoWords(bagOfWords1))), 1); + assert.strictEqual( + computeScore(new Set(splitIntoWords(bagOfWords1)), new Set(splitIntoWords(bagOfWords3))), + 5 / 6 + ); + assert.strictEqual( + computeScore(new Set(splitIntoWords(bagOfWords1)), new Set(splitIntoWords(bagOfWords4))), + 1 / 9 + ); + assert.strictEqual( + computeScore(new Set(splitIntoWords(bagOfWords1)), new Set(splitIntoWords(bagOfWords5))), + 1 / 9 + ); + }); + + test('Snippets never overlap, the highest score wins.', async function () { + // When overlapping snippets are found, the snippet with the highest score wins and the others are dropped, e.g.: + // given the ref doc of "the speed of light is incredibly fast", the doc "the light is incredibly fast" matches + // with score 0.75, but the next "The speed of light is incredibly fast" matches with score 1, so the previous overlapping + // snippet is dropped. + const windowLength = 2; + const doc1 = { + source: 'the light\nis incredibly fast\nthe speed of light\nis incredibly fast\nexcessively bright, the morning sun\n was hot casting elongated shadows', + uri: 'file:///home/user/test.js', + }; + const refDoc = { + source: 'the speed of light\nis incredibly fast', + languageId: '', + uri: 'file:///home/user/test2.js', + }; + assert.deepStrictEqual( + await retrieveAllSnippetsWithJaccardScore(doc1, refDoc, windowLength, SortOptions.None), + [ + { score: 1, startLine: 1, endLine: 3 }, + { score: 0.25, startLine: 3, endLine: 5 }, + ] + ); + }); +}); + +suite('Test getSimilarSnippets function', function () { + const docSource: string = dedent` + A + B + C + D| + E + F + G`; + const doc: DocumentInfoWithOffset = { + relativePath: 'source1', + uri: 'source1', + source: docSource, + languageId: 'python', + offset: docSource.indexOf('|'), // reference snippet will be A B C D + }; + + const similarFiles: SimilarFileInfo[] = [ + { + relativePath: 'similarFile1', + uri: 'similarFile1', + source: dedent` + A + B + C + H + X + Y + Z + `, + }, + { + relativePath: 'similarFile2', + uri: 'similarFile2', + source: dedent` + D + H + `, + }, + ]; + + setup(async function () { + await initializeTokenizers; + }); + + test('Returns correct snippet in conservative mode', async function () { + const options: SimilarFilesOptions = conservativeFilesOptions; + + const snippetLocations = (await getSimilarSnippets(doc, similarFiles, options)).map(snippet => [ + snippet.startLine, + snippet.endLine, + ]); + const correctSnippetLocations: number[][] = [ + [0, 7], // A B C H X Y Z + ]; + assert.deepStrictEqual(snippetLocations, correctSnippetLocations); + }); + test('Returns correct snippets in eager mode', async function () { + const options: SimilarFilesOptions = defaultSimilarFilesOptions; + const snippetLocations = (await getSimilarSnippets(doc, similarFiles, options)).map(snippet => [ + snippet.startLine, + snippet.endLine, + ]); + const correctSnippetLocations: number[][] = [ + [0, 7], // A B C H X Y Z + [0, 2], // D H - included as get up to 4 similar docs + ]; + assert.deepStrictEqual(snippetLocations.sort(), correctSnippetLocations.sort()); + }); + test('Returns no snippet in None mode', async function () { + const options: SimilarFilesOptions = nullSimilarFilesOptions; + const snippetLocations = (await getSimilarSnippets(doc, similarFiles, options)).map(snippet => [ + snippet.startLine, + snippet.endLine, + ]); + const correctSnippetLocations: number[][] = []; + assert.deepStrictEqual(snippetLocations, correctSnippetLocations); + }); +}); + +suite('Test trimming reference document', function () { + const docSource: string = dedent` + 1 + 2 + 3 + 4 + 5 + 6| + 7`; + const doc: DocumentInfoWithOffset = { + relativePath: 'source1', + uri: 'source1', + source: docSource, + languageId: 'python', + offset: docSource.indexOf('|'), + }; + + test('FixedWindowSizeJaccardMatcher trims reference document correctly', async function () { + for (let windowLength = 1; windowLength < 7; windowLength++) { + const matcherFactory = FixedWindowSizeJaccardMatcher.FACTORY(windowLength); + const matcher = matcherFactory.to(doc); + const referenceTokens = [...(await matcher.referenceTokens)]; + // Don't get 7 because it's after the cursor + const correctReferenceTokens: string[] = ['1', '2', '3', '4', '5', '6'].slice(-windowLength); + assert.deepStrictEqual(referenceTokens, correctReferenceTokens); + } + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/snippets.test.ts b/completions-sample-code/vscode-node/prompt/src/test/snippets.test.ts new file mode 100644 index 0000000..d8f11fd --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/snippets.test.ts @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { SnippetProviderType, SnippetSemantics, announceSnippet } from '../snippetInclusion/snippets'; +import * as assert from 'assert'; +import dedent from 'ts-dedent'; + +suite('Unit tests for snippet.ts', () => { + const bogusSnippet = { + relativePath: 'snippet1.ts', + score: 1.0, + startLine: 1, + endLine: 3, + provider: SnippetProviderType.Path, + semantics: SnippetSemantics.Snippet, + snippet: dedent` + A + B + C`, + }; + + test('announceSnippet', function () { + assert.deepStrictEqual(announceSnippet(bogusSnippet), { + headline: 'Compare this snippet from snippet1.ts:', + snippet: dedent` + A + B + C`, + }); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/subsetMatching.test.ts b/completions-sample-code/vscode-node/prompt/src/test/subsetMatching.test.ts new file mode 100644 index 0000000..c426f5f --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/subsetMatching.test.ts @@ -0,0 +1,367 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import assert from 'assert'; +import { DocumentInfoWithOffset, SimilarFileInfo } from '../prompt'; +import { + defaultSimilarFilesOptions, + getSimilarSnippets, + SimilarFilesOptions, +} from '../snippetInclusion/similarFiles'; +import { SnippetWithProviderInfo } from '../snippetInclusion/snippets'; + +async function findAndScoreBlocks( + referenceDoc: DocumentInfoWithOffset, + relatedFiles: SimilarFileInfo[], + useSubsetMatching: boolean +): Promise<SnippetWithProviderInfo[]> { + const options: SimilarFilesOptions = { + snippetLength: defaultSimilarFilesOptions.snippetLength, + threshold: defaultSimilarFilesOptions.threshold, + maxTopSnippets: defaultSimilarFilesOptions.maxTopSnippets, + maxCharPerFile: defaultSimilarFilesOptions.maxCharPerFile, + maxNumberOfFiles: defaultSimilarFilesOptions.maxNumberOfFiles, + maxSnippetsPerFile: defaultSimilarFilesOptions.maxSnippetsPerFile, + useSubsetMatching, + }; + + return getSimilarSnippets(referenceDoc, relatedFiles, options); +} + +function fileScore(snippets: SnippetWithProviderInfo[], partialFileName: string): number { + for (const snippet of snippets) { + if (snippet.relativePath?.indexOf(partialFileName) !== -1) { + return snippet.score; + } + } + + assert(false, 'Expected valid file name'); +} + +suite('Similar files with subset matching Test Suite', function () { + /** + * This test ensures that only tokens from the current method are used when + * computing the score for a chunk. + * + * Compare this to @see FixedWindowSizeJaccardMatcher + * which would use any tokens in the same 60-line-delimited chunk of code that + * the caret fell in. + * + * Scenarios where the caret is in a sub-60-line methods or near a 60 line chunk + * seam would end up getting results that are more related to the neighboring + * methods. + */ + test('Only current method is considered as part of reference tokens', async function () { + const file0 = ` + public static class TestClass + { + public static void UnrelatedMethod(IBar bar) + { + var thing = UnrelatedThing(); + thing.DistractingMethodName(); + } + + public static void Foo(IBar bar) + { + var service = bar.GetService(typeof(IBaz)); + + service.DoTheThing(); + | + } + + public static void UnrelatedMethod2(IBar bar) + { + // This method is unrelated but can DoTheThing to a service + } + } + `; + + const file1 = ` + public interface IBar + { + public object GetService(Type type); + } + `; + + const file2 = ` + public interface IBaz + { + public static void DoTheThing(); + } + `; + + const file3 = ` + public static class DistractionClass + { + public DistractionClass UnrelatedThing() + { + TestClass.UnrelatedMethod(null); + + UnrelatedMethod(null); + } + + public void DistractingMethodName() + { + TestClass.UnrelatedMethod2(null); + } + } + `; + + // ********************************************************** + // Score with the old 60-line-delimited reference token chunk. + const oldScores = await findAndScoreBlocks( + { source: file0, uri: 'file:///home/user/file0.js', languageId: 'csharp', offset: file0.indexOf('|') }, + [ + { source: file1, uri: 'file:///home/user/file1.js', relativePath: 'file1' }, + { source: file2, uri: 'file:///home/user/file2.js', relativePath: 'file2' }, + { source: file3, uri: 'file:///home/user/file3.js', relativePath: 'file3' }, + ], + false + ); + + // We expect the old way to prefer the distraction class, which has lots of terms that look like stuff from + // the neighboring methods. + assert( + fileScore(oldScores, 'file3') > fileScore(oldScores, 'file2') && + fileScore(oldScores, 'file2') > fileScore(oldScores, 'file1'), + 'Expected 60-line-delimited reference chunks to prefer the distraction class because it resembles neighboring methods' + ); + // ********************************************************** + + // ********************************************************** + // Score with the new subset matching mechanism. + const newScores = await findAndScoreBlocks( + { source: file0, uri: 'file:///home/user/file0.js', languageId: 'csharp', offset: file0.indexOf('|') }, + [ + { source: file1, uri: 'file:///home/user/file1.js', relativePath: 'file1' }, + { source: file2, uri: 'file:///home/user/file2.js', relativePath: 'file2' }, + { source: file3, uri: 'file:///home/user/file3.js', relativePath: 'file3' }, + ], + true + ); + + // We expect the new way to prefer the second file because it contains the most tokens that match + // the method enclosing the caret. + assert( + fileScore(newScores, 'file2') > fileScore(newScores, 'file1') && + fileScore(newScores, 'file1') > fileScore(newScores, 'file3'), + 'Expected that the file containing IBaz interface would be the best match' + ); + // ********************************************************** + }); + + /** + * This test ensures that methods are matched only based on the tokens from the reference + * chunk that they do contain and are not penalized for containing additional tokens that + * don't appear in the reference set. + * + * Compare this to @see FixedWindowSizeJaccardMatcher which would use Jaccard similarity to + * score. Jaccard similarity gives preferences to chunks with sets of identical tokens. + * + * Intuitively, scenarios where a token is a type or method reference get penalized because + * they have tokens in common for the name of the method but have divergent content. + */ + test('Methods are not penalized for being supersets of the reference chunk', async function () { + const file0 = ` + public static class TestClass + { + public static void Foo(Bar bar) + { + bar.Baz(); + | + } + } + `; + + const file1 = ` + public class Bar + { + public void Baz() + { + // This method has a bunch of extra tokens that don't match file0 and collectively + // reduce its score relative to the other files. + } + } + `; + + const file2 = ` + public class Bar2 + { + public void Baz() + { + } + } + `; + + const file3 = ` + public class Bar3 + { + public void Baz3() + { + } + } + `; + + // ********************************************************** + // Score with the old 60-line-delimited reference token chunk. + const oldScores = await findAndScoreBlocks( + { source: file0, uri: 'file:///home/user/file0.js', languageId: 'csharp', offset: file0.indexOf('|') }, + [ + { source: file1, uri: 'file:///home/user/file1.js', relativePath: 'file1' }, + { source: file2, uri: 'file:///home/user/file2.js', relativePath: 'file2' }, + { source: file3, uri: 'file:///home/user/file3.js', relativePath: 'file3' }, + ], + false + ); + + // We expect the old way to prefer the simpler code samples, even when they match fewer tokens, + // because there are fewer non-matching additional tokens. + assert( + fileScore(oldScores, 'file2') > fileScore(oldScores, 'file1') && + fileScore(oldScores, 'file1') === fileScore(oldScores, 'file3'), + 'Expected 60-line-delimited reference chunks to prefer the distraction class because it resembles neighboring methods' + ); + // ********************************************************** + + // ********************************************************** + // Score with the new method. + const newScores = await findAndScoreBlocks( + { source: file0, uri: 'file:///home/user/file0.js', languageId: 'csharp', offset: file0.indexOf('|') }, + [ + { source: file1, uri: 'file:///home/user/file1.js', relativePath: 'file1' }, + { source: file2, uri: 'file:///home/user/file2.js', relativePath: 'file2' }, + { source: file3, uri: 'file:///home/user/file3.js', relativePath: 'file3' }, + ], + true + ); + + // We expect the new way to prefer the file with matching class and method names because we're no longer + // penalizing samples for having different tokens. + assert( + fileScore(newScores, 'file1') > fileScore(newScores, 'file2') && + fileScore(newScores, 'file2') > fileScore(newScores, 'file3'), + 'Expected subset matching method to prefer the file with the most token matches' + ); + // ********************************************************** + }); + + /** + * This test ensures that only tokens from the current class are used when + * computing the score for a chunk. + * + * Compare this to @see FixedWindowSizeJaccardMatcher + * which would use any tokens in the same 60-line-delimited chunk of code that + * the caret fell in. + * + * Scenarios where the caret is in a sub-60-line method or near a 60 line chunk + * seam would end up getting results that are more related to the neighboring + * methods. + */ + test('Only current class is considered as part of reference tokens', async function () { + const file0 = ` + + public static class TestClass2 + { + public static void UnrelatedMethod(IBar bar) + { + var thing = UnrelatedThing(); + thing.DistractingMethodName(); + } + } + + public static class TestClass + { + public static void Foo(IBar bar) + { + var service = bar.GetService(typeof(IBaz)); + + service.DoTheThing(); + } + + | + } + + public static class TestClass3 + { + public static void UnrelatedMethod2(IBar bar) + { + // This method is unrelated but can DoTheThing to a service + } + } + `; + + const file1 = ` + public interface IBar + { + public object GetService(Type type); + } + `; + + const file2 = ` + public interface IBaz + { + public static void DoTheThing(); + } + `; + + const file3 = ` + public static class DistractionClass + { + public DistractionClass UnrelatedThing() + { + TestClass.UnrelatedMethod(null); + + UnrelatedMethod(null); + } + + public void DistractingMethodName() + { + TestClass.UnrelatedMethod2(null); + } + } + `; + + // ********************************************************** + // Score with the old 60-line-delimited reference token chunk. + const oldScores = await findAndScoreBlocks( + { source: file0, uri: 'file:///home/user/file0.js', languageId: 'csharp', offset: file0.indexOf('|') }, + [ + { source: file1, uri: 'file:///home/user/file1.js', relativePath: 'file1' }, + { source: file2, uri: 'file:///home/user/file2.js', relativePath: 'file2' }, + { source: file3, uri: 'file:///home/user/file3.js', relativePath: 'file3' }, + ], + false + ); + + // We expect the old way to prefer the distraction class, which has lots of terms that look like stuff from + // the neighboring methods. + assert( + fileScore(oldScores, 'file3') > fileScore(oldScores, 'file2') && + fileScore(oldScores, 'file2') > fileScore(oldScores, 'file1'), + 'Expected 60-line-delimited reference chunks to prefer the distraction class because it resembles neighboring methods' + ); + // ********************************************************** + + // ********************************************************** + // Score with the new subset matching mechanism. + const newScores = await findAndScoreBlocks( + { source: file0, uri: 'file:///home/user/file0.js', languageId: 'csharp', offset: file0.indexOf('|') }, + [ + { source: file1, uri: 'file:///home/user/file1.js', relativePath: 'file1' }, + { source: file2, uri: 'file:///home/user/file2.js', relativePath: 'file2' }, + { source: file3, uri: 'file:///home/user/file3.js', relativePath: 'file3' }, + ], + true + ); + + // We expect the new way to prefer the second file because it contains the most tokens that match + // the method enclosing the caret. + assert( + fileScore(newScores, 'file2') > fileScore(newScores, 'file3') && + fileScore(newScores, 'file3') === fileScore(newScores, 'file1'), + 'Expected that the file containing IBaz interface would be the best match' + ); + // ********************************************************** + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/suffixmatch.test.ts b/completions-sample-code/vscode-node/prompt/src/test/suffixmatch.test.ts new file mode 100644 index 0000000..97cacee --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/suffixmatch.test.ts @@ -0,0 +1,17 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { findEditDistanceScore } from '../suffixMatchCriteria'; +import * as assert from 'assert'; + +suite('EditDistanceScore Test Suite', function () { + test('findEditDistanceScore computes correct score of two number[]', function () { + assert.strictEqual(findEditDistanceScore([], [])?.score, 0); + assert.strictEqual(findEditDistanceScore([1], [1])?.score, 0); + assert.strictEqual(findEditDistanceScore([1], [2])?.score, 1); + assert.strictEqual(findEditDistanceScore([1], [])?.score, 1); + assert.strictEqual(findEditDistanceScore([], [1])?.score, 1); + assert.strictEqual(findEditDistanceScore([1, 2, 3], [3, 2, 1])?.score, 2); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/testHelpers.ts b/completions-sample-code/vscode-node/prompt/src/test/testHelpers.ts new file mode 100644 index 0000000..60d8967 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/testHelpers.ts @@ -0,0 +1,59 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { describeTree, IndentationTree, isLine, VirtualNode } from '../indentation'; +import * as assert from 'assert'; + +/** + * Asserts that two trees are isomorphic. + * @param actual The tree to test. + * @param expected The tree expected to be equal (source lines can be abbreviated with '...'). + * @param strictness Should the tree be deeply equal (including indentation and line numbers), + * or is in enough for the children and types of each node match? + * @param treeParent The tree's parent for context (optional) + * @param parentIndex The index for the tree in its parent's subs (optional) + */ + +export function compareTreeWithSpec<T>( + actual: IndentationTree<T>, + expected: IndentationTree<T>, + strictness: 'strict' | 'structure' = 'strict', + treeParent?: IndentationTree<T>, + parentIndex?: number +) { + if (actual.type !== expected.type) { + failCompare( + actual, + expected, + `type of tree doesn't match, ${actual.type} ${expected.type}`, + treeParent, + parentIndex + ); + } + if (actual.subs.length !== expected.subs.length) { + failCompare(actual, expected, 'number of children do not match', treeParent, parentIndex); + } + + if (strictness === 'strict' && isLine(actual)) { + if (actual.indentation !== (expected as VirtualNode<T>).indentation) { + failCompare(actual, expected, `virtual node indentation doesn't match`, treeParent, parentIndex); + } + } + + for (let i = 0; i < actual.subs.length; ++i) { + compareTreeWithSpec(actual.subs[i], expected.subs[i], strictness, actual, i); + } +} + +function failCompare<T>( + tree: IndentationTree<T>, + expected: IndentationTree<T>, + reason: string, + treeParent?: IndentationTree<T>, + parentIndex?: number +) { + assert.fail(`Reason: ${reason} + Tree: ${describeTree(tree)} + Expected: ${describeTree(expected)}`); +} diff --git a/completions-sample-code/vscode-node/prompt/src/test/testdata/example.py b/completions-sample-code/vscode-node/prompt/src/test/testdata/example.py new file mode 100644 index 0000000..c7ebe0a --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/testdata/example.py @@ -0,0 +1,501 @@ +""" +This is an example Python source file to use as test data. It's pulled from the synth repo +with minor edits to make it a better test case. +""" + +from tree_sitter import Language, Parser +import re +import os, sys +from dataclasses import dataclass, field +import codesynthesis.synthesis as synthesis +import harness.fun_run as fun_run +from harness.utils import temporary_path_change_to, get_canonical_logger, TreeSitter +import types +import subprocess + +logger = get_canonical_logger("FilesWithImport") + +@dataclass +class Import: + module: str + filename: str = None + source_text: str = None + as_name: str = None + original_statement: str = None + # list of imported objects + imported: list = field(default_factory=list) + + def add_source(self): + assert self.filename, f"No filename for {self.module}" + with open(self.filename, "r") as f: + self.source_text = f.read() # intentional trailing whitespace + + +class ImportAnalysis: + """ + Methods to discribe the effect of an import statement. + """ + def __init__(self, imp: Import, clean_environment=None, import_directly=True): + self.imp = imp + # a clean dict into which to import the module + if clean_environment is None: + clean_environment = { + "__builtins__": __builtins__, + "__name__": __name__, + "__doc__": __doc__, + "__package__": "thefuck", + "__file__": __file__ + } + self._clean_environment = clean_environment + if import_directly: + self.impact = self._import(imp.original_statement, imp.filename) + self.imported_objects = self._get_imported_objects() + else: + self.impact = None + self.imported_objects = None + self.imported_directly = import_directly # if False, then can only use describe_from_outside + + def _import(self, statement, filename): + """ + tries to import module from filename path, if that doesn't work parent folder, etc., + returns a dictionary of the new objects created by the import statement. + """ + path = os.path.dirname(filename) + d = self._clean_environment.copy() + while len(path) > 1: + try: + # chdir to path, then exec statement + #with temporary_path_change_to(path): + d_before = d.copy() + exec(statement, d) + # remove from d what was already in d_before + d = {k: v for k, v in d.items() if k not in d_before} + return d + except ModuleNotFoundError: + path = os.path.dirname(path) + raise FileNotFoundError(f"Unable to import module from {filename}") + + def _get_imported_objects(self, include_private=False): + """ + Returns a dict of relevant imported objects (by their names) + For `from a import b`, this should be {"b": b} + For `import a as b`, this should be {"b.x": b.x} for all imported objects x + For `from a import *`, this should be {"x": x} for all imported objects x + TODO For `import a.b`, this should be {"a.b.x": a.b.x} -- could maybe just increase depth correctly? + """ + out = {} + depth = 0 + while all(isinstance(is_it_a_module, types.ModuleType) for is_it_a_module in out.values()): + out = self._unpack_dict(self.impact, include_private=include_private, depth=depth) + depth += 1 + return out + + @classmethod + def _unpack_dict(cls, dictionary, include_private, depth, prefix=""): + # names imported directly, except modules that will be expanded on later, and perhaps private names + out = {prefix + name: obj for name, obj in dictionary.items() if \ + (depth == 0 or not isinstance(obj, types.ModuleType)) and \ + (include_private or not name.startswith("_"))} + + if depth > 0: + # for modules, add their objects + for name, module in dictionary.items(): + if isinstance(module, types.ModuleType): + one_level_deeper = cls._unpack_dict(module.__dict__, include_private, depth=depth-1, prefix=name+".") + out.update(one_level_deeper) + + return out + + def get_names_of_imported_by_type(self): + """return the imported objects as a dict, with their type name as key""" + d = dict() + for object_name, obj in self.imported_objects.items(): + typ = type(obj).__name__ + if typ in d: + d[typ].append(object_name) + else: + d[typ] = [object_name] + return d + + def get_methods_of_type(self, class_name, include_private=False): + """return the names of methods of the given type""" + cls = self.imported_objects[class_name] + names = [name for name in dir(cls) if include_private or not name.startswith("_")] + # return only the names which describe methods + return [name for name in names if callable(getattr(cls, name))] + + def get_function_description(self, func_name): + """return a string describing the function and its arguments""" + fun = self.imported_objects[func_name] + args = "(" + ", ".join(fun.__code__.co_varnames[:fun.__code__.co_argcount]) + ")" + return f"{func_name}{args}" + + def get_class_description(self, class_name): + """return a string describing the class and its methods""" + return f"{class_name} with functions {', '.join(self.get_methods_of_type(class_name))}" + + def describe(self) -> str: + """ + return a description of the import, detailing classes, functions and properties as follows: + [as name] adds the following classes: + - [class name] with functions [function name], ..., [another function name] + ... + [as name] adds the functions [function name], ..., [another function name] + [as name] also adds the objects: [property name: property type], ..., [another property name: property type] + """ + imported_names = self.get_names_of_imported_by_type() + if "type" in imported_names.keys(): + classes = imported_names["type"] + answer = f"{self.imp.as_name} adds the following classes:\n" + \ + "\n".join([f" - {self.get_class_description(class_name)}" + for class_name in classes]) + \ + "\n" + else: + answer = "" + + if "function" in imported_names.keys(): + functions = imported_names["function"] + answer += f"{self.imp.as_name} adds the functions {', '.join(self.get_function_description(func_name) for func_name in functions)}\n" + + # all other types are designated as properties + properties = [key for key in imported_names.keys() if key not in ["function", "type"]] + if properties: + answer += f"{self.imp.as_name} {'also ' if answer else ''}adds the objects: {', '.join([f'{imported_names[name]}: {name}' for name in properties])}\n" + + return answer + + def description_comment(self) -> str: + """wrap the multiline string describe() into a comment, each of whose lines begins with '#'""" + return "\n".join(["# " + line for line in self.describe().split("\n")]) + + def describe_from_the_outside(self, path): + """writes the import statement plus ask to describe to a file in path, runs that file, gathers output""" + filename = os.path.abspath(path + "/" + "tmp.py") + assert os.path.exists(filename) == False, f"{filename} already exists!" + try: + with open(filename, "w") as f: + f.write(f"""from codesynthesis.files_with_import import ImportAnalysis, ImportParser +imp_statement = "{self.imp.original_statement}" +imp = ImportParser().get_all_imports(imp_statement, "{filename}")[0] +env = dict( + __builtins__= __builtins__, + __name__= __name__, + __doc__= __doc__, + __package__= __package__, + __file__= __file__ + ) +analysis = ImportAnalysis(imp, env) +print(analysis.describe())""" + ) + # get module name from cwd and filename: + relative_filename = os.path.relpath(filename, os.getcwd()) + module_name = relative_filename.replace(".py", "").replace("/", ".") + try: + out = subprocess.check_output(["python", "-m", module_name], stderr=subprocess.STDOUT).decode("utf-8").strip() + except subprocess.CalledProcessError as e: + error = e.output.decode("utf-8") + if "FileNotFoundError: Unable to import module from" in error: + logger.error(f"Could not import module through {self.imp.original_statement} in {filename}. The import of {self.imp.module} will not be documented.") + return "" + else: + raise e + finally: + os.remove(filename) + return out + + def describe_from_the_outside_as_comment(self, path): + return "\n".join(["# " + line for line in self.describe_from_the_outside(path).split("\n")]) + +class ImportParser: + PY_LANGUAGE = TreeSitter().language("python") + IMP_QUERY = ["(import_statement) @import", + "(future_import_statement) @import", + "(import_from_statement) @import"] + MODULE_LEVEL_IMP_QUERY = ["(module (import_statement) @import)", + "(module (future_import_statement) @import)", + "(module (import_from_statement) @import)"] + # TODO: Define MODULE_SCOPE_IMP_QUERY, where the import can't be inside a functon, but can be inside an if or try. + def __init__(self): + self.parser = TreeSitter().get_parser("python") + Parser() + self.parser.set_language(self.PY_LANGUAGE) + + @staticmethod + def get_text_from(text, capture): + """ + Trim the text to the content corresponding to a single tree-sitter capture expression. + @param text: The whole text of the language document. + @param capture: The particular capture expression within the document to trim to. + @return: The text for the capture expression only. + """ + lines = text.split('\n') + relevant_lines = lines[capture.start_point[0] : capture.end_point[0]+1] + # in case the extract is just one line, the trim on the right needs to come before the trim on the left! + relevant_lines[-1] = relevant_lines[-1][:capture.end_point[1]] + relevant_lines[0] = relevant_lines[0][capture.start_point[1]:] + return '\n'.join(relevant_lines) + + @staticmethod + def replace_text_from(text, capture, replacement): + """ + replaces the text from the capture with the replacement. + """ + lines = text.split('\n') + prelude = lines[0 : capture.start_point[0]+1] + if prelude: + prelude[-1] = prelude[-1][:capture.start_point[1]+1] + postlude = lines[capture.end_point[0] + 1 :] + if postlude: + postlude[0] = postlude[0][capture.end_point[1]:] + return '\n'.join(prelude + [replacement] + postlude) + + + def get_list_of_import_captures(self, text): + tree = self.parser.parse(bytes(text, "utf8")) + list_of_capture_lists = [self.PY_LANGUAGE.query(query).captures(tree.root_node) for query in self.IMP_QUERY] + # flatten array + return [item[0] for sublist in list_of_capture_lists for item in sublist] + + def get_import_statements(self, text): + """ + returns a list of all imports (as far as found in the statement, not the background like content) + """ + captures = self.get_list_of_import_captures(text) + imports = [self.parse_single_import(self.get_text_from(text, capture)) for capture in captures] + return imports + + def parse_single_import(self, relevant_text): + """ + parses a single import statement (without background like content) + TODO: deal with the case of several imports in one expression, e.g. import a, b, c + """ + # module is the XXX in from XXX import ..., or in import XXX + if re.search('from ([^ ]+) import', relevant_text): + module = re.search('from (\\.)*([^ ]+) import', relevant_text).group(2) + else: + module = re.search('import (\\.)*([^ ]+)', relevant_text).group(2) + # as_name is the XXX in import ... as XXX or from ... import ... as XXX + if re.search(' as ([^ ]+)', relevant_text): + as_name = re.search(' as ([^ ]+)', relevant_text).group(1) + else: + as_name = module + # imported are the XXX, YYY, ZZZ in from XXX import XXX, YYY, ZZZ + # but we don't need them right now, so TODO + imported = ["TODO"] + return Import(module=module, as_name=as_name, imported=imported, original_statement=relevant_text) + + def get_all_imports(self, source_text, source_filename): + """ + adds all import files found in the source_text, if it were at disk as filename + skips standard packages (i.e. anything not found on disk at location filename) + """ + raw_imports = self.get_import_statements(source_text) + relevant_imports = [] + for imp in raw_imports: + # check whether filename exists; if not: check whether it exist in the parent folder, recursively + base_folder = os.path.dirname(source_filename) + # Note: The following does not take '..module' imports into account differently + # (which will mostly be ok though, unless there's a name clash) + while filename:= base_folder + '/' + '/'.join(imp.module.split('.')): + if os.path.isfile(filename + '.py'): + imp.filename = filename + '.py' + imp.add_source() + relevant_imports.append(imp) + break + else: + base_folder = os.path.dirname(base_folder) + if len(base_folder) <= 1: + break + return relevant_imports + + def remove_imports(self, text, imps): + """ + returns the text minus the imports + + Note: This is theoretically a bit too aggressive, as it also removes the text of the import statements inside quotes etc + Note: This is theoreticallly a bit too aggressive, as it also removes captures where the import model name is only a substring + """ + captures = self.get_list_of_import_captures(text) + for capture in captures: + # only act if imp.module is in the capture expression + capture_text = self.get_text_from(text, capture) + if any(imp.module in capture_text for imp in imps): + text = text.replace(capture_text, '') + return text + + def truncate_left_but_keep_module_level_imports(self, text, length_in_tokens: int, fixed_prefix: str = ""): + """make sure to keep the module level imports, but otherwise drop lines as needed, calling truncate_left""" + # identify imports + captures = self.get_list_of_import_captures(text) + lines = text.split('\n') + idc_to_keep = set() + for capture in captures: + for lineno in range(capture.start_point[0], capture.end_point[0] + 1): + idc_to_keep.add(lineno) + + result = synthesis.truncate_left_keeping_lines_with_preference(lines, idc_to_keep, length_in_tokens, fixed_prefix) + return result + + +def test_import_parser(): + ip = ImportParser() + assert len(ip.get_all_imports("from codesynthesis.synthesis import abc\nprint(32)", "/Users/wunderalbert/openai/synth/test.py")) > 0 + assert len(ip.get_all_imports("import codesynthesis.synthesis as abc\nprint(32)", "/Users/wunderalbert/openai/synth/test.py")) > 0 + +class FunctionWithImportsKept(fun_run.PythonFunctionInTheWild): + def make_prompt_for_fct(self, max_length_in_tokens = synthesis.CONTEXT_WINDOW_SIZE): + return ImportParser().truncate_left_but_keep_module_level_imports('# Python 3\n' + self.prelude + '\n' + self.header, max_length_in_tokens) + + +class FunctionWithImports(fun_run.PythonFunctionInTheWild): + def __init__(self, function_location, discriminative_model): + super().__init__(function_location, discriminative_model) + # get the absolute path + self.filename = os.path.abspath(self.function_location.path) + source_text = ''.join(self.source_lines) + self.imports = ImportParser().get_all_imports(source_text, self.filename) + importless_source_lines_without_newline_char = ImportParser().remove_imports(source_text, self.imports).split("\n") + self.importless_source_lines = [line + "\n" for line in importless_source_lines_without_newline_char] + + def make_prompt_for_fct_without_imports(self): + """call super.make_prompt_for_function, but with importless_source_lines temporarily replacing source_lines""" + complete_source_lines = self.source_lines + try: + self.source_lines = self.importless_source_lines + prompt = super().make_prompt_for_fct(fun_run.VERY_LARGE_NUMBER) + finally: + self.source_lines = complete_source_lines + return prompt + + +class FunctionWithImportsPastedVerbatim(FunctionWithImports): + """ + Pastes code verbatim + """ + def make_prompt_for_fct(self, max_length_in_tokens): + super_prompt = super().make_prompt_for_fct_without_imports() + desired_prompt = "\n\n".join([imp.source_text for imp in self.imports]) + "\n\n" + super_prompt + truncated_prompt = synthesis.truncate_left(desired_prompt, max_length_in_tokens) + return truncated_prompt + +class FunctionWithImportsPastedWithComments(FunctionWithImports): + """ + Pastes code verbatim as a comment that this is the content of that module + """ + def make_prompt_for_fct(self, max_length_in_tokens): + prompt = "" + for imp in self.imports: + prompt = prompt + \ + f"# Content of module {imp.as_name}\n# " + \ + f"\n# ".join(imp.source_text.split('\n')) + \ + f"\n\n" + prompt = prompt + super().make_prompt_for_fct() + truncated_prompt = synthesis.truncate_left(prompt, max_length_in_tokens) + return truncated_prompt + +class FunctionWithImportsNamespacedInClasses(FunctionWithImports): + """ + Encapsulates imports within classes -- this does not create sound code, as the self argument is missing from functions, and variables are not prefixed with `class.` + """ + def make_prompt_for_fct(self, max_length_in_tokens): + prompt = "" + for imp in self.imports: + prompt = prompt + \ + f"class {imp.as_name}:\n{self.STRING_FOR_INDENTATION_LEVEL_INCREASE}" + \ + f"\n{self.STRING_FOR_INDENTATION_LEVEL_INCREASE}".join(imp.source_text.split('\n')) + \ + f"\n\n" + prompt = prompt + super().make_prompt_for_fct_without_imports() + truncated_prompt = synthesis.truncate_left(prompt, max_length_in_tokens) + return truncated_prompt + + +class FunctionWithImportsReplacedOneByOne(fun_run.PythonFunctionInTheWild): + """ + Schemes where all imports that import local files are replaced or added to, + e.g. by summarizing their content, or quoting it, etc. + """ + def __init__(self, function_location, discriminative_model): + super().__init__(function_location, discriminative_model) + # get the absolute path + self.filename = os.path.abspath(self.function_location.path) + source_text = ''.join(self.source_lines) + self.imports = ImportParser().get_all_imports(source_text, self.filename) + logger.debug(f"In the function {self.function_location.name}, there are {len(self.imports)} repo imports: {[imp.module for imp in self.imports]}") + + keep_imports_and_description_by_preference = True + + def replace_import(self, imp: Import): + """ + Given one import, returns the replacement text pasted into where that import was. + E.g. for a text `foo\nimport bar\nbaz`, if the import is replaced by the string "bat", + the result will be "foo\nbat\nbaz" + """ + raise NotImplementedError("Needs to be implemented in subclass.") + + def make_prompt_for_fct(self, max_length_in_tokens): + """call replace import for each import""" + prompt = super().make_prompt_for_fct(max_length_in_tokens) + # imports should be sorted by start position anyways, but let's be safe + sorted_imports = self.imports.copy() + # replace them from bottom to top, so that the replacements don't change the position of other imports + sorted_imports.reverse() + import_replacements = set() + for imp in sorted_imports: + new_statement = self.replace_import(imp) + import_replacements.add(new_statement) + prompt = prompt.replace(imp.original_statement, new_statement) + # if descriptions have higher priority, extract the lines where they are and pass those lines to synthesis.truncate_left_keeping_lines_with_preference + if self.keep_imports_and_description_by_preference: + new_lines = set() + for statement in import_replacements: + match = prompt.find(statement) + lines_of_match = range( + prompt[:match].count("\n"), + prompt[:match+len(statement)].count("\n")+1) + new_lines.update(lines_of_match) + truncated_prompt = synthesis.truncate_left_keeping_lines_with_preference(prompt.split("\n"), new_lines, max_length_in_tokens) + else: + truncated_prompt = synthesis.truncate_left(prompt, max_length_in_tokens) + return truncated_prompt + + +class FunctionWithImportsCommentedWithTheFunctionsTheyImport(FunctionWithImportsReplacedOneByOne): + """ + adds a comment to each import with the objects it imports + """ + def replace_import(self, imp: Import): + # find all toplevel functions in imp.source_text, i.e. the `foo` from lines of the form `def foo()` + source_starting_with_newline = ("\n" + imp.source_text) + toplevel_functions = re.findall(r"\ndef\s*([a-zA-Z0-9_]+)\s*\(", source_starting_with_newline) + toplevel_classes = re.findall(r"\nclass\s*([a-zA-Z0-9_]+)", source_starting_with_newline) + toplevel_classes_with_their_functions = {} + for classname in toplevel_classes: + cls_source_lines_with_trailing = source_starting_with_newline.split("\nclass " + classname)[1].split("\n")[1:] + if not cls_source_lines_with_trailing: + continue + indices_after_class_end = [i for i, line in enumerate(cls_source_lines_with_trailing) if not line.startswith(" " * 4) and not line.startswith("#")] + if indices_after_class_end: + cls_source_lines = cls_source_lines_with_trailing[:indices_after_class_end[0]] + else: + cls_source_lines = cls_source_lines_with_trailing + cls_body = "\n".join(cls_source_lines) + # remove everything after the first line with less than 4 spaces indentation + functions_not_starting_with_underscore = re.findall(r"\ndef\s*([a-zA-Z0-9_]+)\s*\(", cls_body) + toplevel_classes_with_their_functions[classname] = functions_not_starting_with_underscore + description = imp.original_statement + if toplevel_functions: + description = description + f"\n# module {imp.as_name} declares the following functions: {', '.join(toplevel_functions)}" + for classname, functions in toplevel_classes_with_their_functions.items(): + description = description + f"\n# module {imp.as_name} declares the class {classname}" + + if functions: + description = description + f", which contains the following functions: {', '.join(functions)}" + return description + +class FunctionWithImportsCommentedWithImportAnalysis(FunctionWithImportsReplacedOneByOne): + def replace_import(self, imp: Import): + analysis = ImportAnalysis(imp, import_directly=False) + description = imp.original_statement + "\n" + \ + analysis.describe_from_the_outside_as_comment(os.path.dirname(self.filename)) + logger.debug(f"To the import {imp.module} as {imp.as_name}, we add the following comment: {description}") + return description \ No newline at end of file diff --git a/completions-sample-code/vscode-node/prompt/src/test/testdata/lazy_greet.py b/completions-sample-code/vscode-node/prompt/src/test/testdata/lazy_greet.py new file mode 100644 index 0000000..d422996 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/testdata/lazy_greet.py @@ -0,0 +1,5 @@ +def greet(name: str) -> str: + "Does a simple greeting" + return f"Hello {name}" + +greet() diff --git a/completions-sample-code/vscode-node/prompt/src/test/testdata/testTokenizer.ts b/completions-sample-code/vscode-node/prompt/src/test/testdata/testTokenizer.ts new file mode 100644 index 0000000..d0cc402 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/testdata/testTokenizer.ts @@ -0,0 +1,11 @@ +// This is a test file for the sake of testing actual file reads +// We had silently failing tests in the past due to improper +// file spoofing + +export interface Tokenizer { + /** + * Returns the tokenization of the input string as a list of integers + * representing tokens. + */ + tokenize(text: string): Array<number>; +} diff --git a/completions-sample-code/vscode-node/prompt/src/test/testdata/testWishlist.ts b/completions-sample-code/vscode-node/prompt/src/test/testdata/testWishlist.ts new file mode 100644 index 0000000..13ead25 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/testdata/testWishlist.ts @@ -0,0 +1,14 @@ +// This is a test file for the sake of testing actual file reads +// We had silently failing tests in the past due to improper +// file spoofing + +import { Tokenizer } from './testTokenizer'; + +export class PromptWishlist { + /** + * An object to keep track of a list of desired prompt elements, + * and assemble the prompt text from them. + * @param lineEndingOption The line ending option to use + */ + constructor(_tokenizer: Tokenizer) { } +} diff --git a/completions-sample-code/vscode-node/prompt/src/test/tokenizer.test.ts b/completions-sample-code/vscode-node/prompt/src/test/tokenizer.test.ts new file mode 100644 index 0000000..934b27d --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/tokenizer.test.ts @@ -0,0 +1,595 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import * as fs from 'fs'; +import { resolve } from 'path'; +import { ApproximateTokenizer, getTokenizer, TokenizerName } from '../tokenization'; + +// Read the source files and normalize the line endings +const source = fs.readFileSync(resolve(__dirname, 'testdata/example.py'), 'utf8').replace(/\r\n?/g, '\n'); + +suite('Tokenizers can be loaded', function () { + for (const tokenizer of Object.values(TokenizerName)) { + test(`Tokenizer ${tokenizer} can be loaded`, function () { + getTokenizer(tokenizer); + }); + } +}); + +// test suite for MockTokenizer +suite('MockTokenizer', function () { + const tokenizer = getTokenizer(TokenizerName.mock); + + test('tokenize', function () { + const tokens = tokenizer.tokenize('a b c'); + assert.strictEqual(tokens.length, 5); + + for (const token of tokens) { + assert.strictEqual(typeof token, 'number'); + } + }); + + test('detokenize', function () { + const tokens = tokenizer.tokenize('a b c'); + const text = tokenizer.detokenize(tokens); + // unfortunately the mock tokenizer doesn't correctly round-trip the text + // because the token representation is a number. If this matters then we'll + // have to change the mock tokenizer to use a different representation. + assert.strictEqual(text, '97 32 98 32 99'); + }); + + test('tokenLength', function () { + assert.strictEqual(tokenizer.tokenLength('a b c'), 5); + }); + + test('takeFirstTokens', function () { + const tokens = tokenizer.takeFirstTokens('a b c', 3); + assert.strictEqual(tokens.text, 'a b'); + assert.strictEqual(tokens.tokens.length, 3); + }); + + test('takeLastTokens', function () { + const tokens = tokenizer.takeLastTokens('a b c', 3); + assert.strictEqual(tokens.text, 'b c'); + }); + + test('takeLastLinesTokens', function () { + const tokens = tokenizer.takeLastLinesTokens('a b c', 3); + assert.strictEqual(tokens, 'b c'); + }); +}); + +suite('Tokenizer Test Suite - cl100k', function () { + const tokenizer = getTokenizer(TokenizerName.cl100k); + + test('empty string', function () { + const str = ''; + assert.deepStrictEqual(tokenizer.tokenize(str), []); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('space', function () { + const str = ' '; + assert.deepStrictEqual(tokenizer.tokenize(str), [220]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('tab', function () { + const str = '\t'; + assert.deepStrictEqual(tokenizer.tokenize(str), [197]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('simple text', function () { + const str = 'This is some text'; + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + assert.deepStrictEqual(tokenizer.tokenize(str), [2028, 374, 1063, 1495]); + }); + + test('multi-token word', function () { + const str = 'indivisible'; + assert.deepStrictEqual(tokenizer.tokenize(str), [485, 344, 23936]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('emojis', function () { + const str = 'hello ๐Ÿ‘‹ world ๐ŸŒ'; + assert.deepStrictEqual(tokenizer.tokenize(str), [15339, 62904, 233, 1917, 11410, 234, 235]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('contractions', function () { + const str = `you'll`; + assert.deepStrictEqual(tokenizer.tokenize(str), [9514, 3358]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('assert that consecutive newline is never tokenized as multiple newlines', function () { + // This is due to a regular expression change in the tokenizer. + + // Loop through all possible ascii numbers and letters + for (let i = 0; i < 128; i++) { + const char = String.fromCharCode(i); + if (char !== '\n') { + assert.deepStrictEqual(tokenizer.tokenLength(`\n\n${char}`), 2); + } + } + + // Test special characters + assert.deepStrictEqual(tokenizer.tokenize('\n\n๐Ÿ‘‹'), [271, 9468, 239, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n '), [271, 220]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n ๐Ÿ‘‹'), [271, 62904, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n\t'), [271, 197]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n\r'), [271, 201]); + + // New lines are treated specially tho + for (let i = 1; i < 10; i++) { + assert.deepStrictEqual(tokenizer.tokenLength('\n'.repeat(i)), 1); + } + }); + + test('tokenizeStrings', function () { + const tokens_s = tokenizer.tokenizeStrings(source); + assert.strictEqual(tokens_s.join(''), source, 'tokenizeStrings does not join to form the input string'); + const tokens = tokenizer.tokenize(source); + assert.strictEqual(tokens_s.length, tokens.length, 'tokenizeStrings should have same length as tokenize'); + const half = Math.floor(tokens_s.length / 2); + assert.strictEqual( + tokens_s.slice(0, half).join(''), + tokenizer.detokenize(tokens.slice(0, half)), + 'tokenizeStrings slice should represent the corresponding slice with tokenize' + ); + }); + + test('takeLastTokens invariant of starting position', function () { + const suffix = tokenizer.takeLastTokens(source, 25); + assert.strictEqual( + suffix.text, + `"To the import {imp.module} as {imp.as_name}, we add the following comment: {description}")\n return description` + ); + assert.strictEqual(suffix.tokens.length, 25); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(50), 25).text); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(100), 25).text); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(150), 25).text); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(200), 25).text); + }); + + test('takeLastTokens returns the desired number of tokens', function () { + assert.strictEqual(tokenizer.takeLastTokens(source, 30).tokens.length, 30); + assert.strictEqual(tokenizer.takeLastTokens(source, 29).tokens.length, 29); + assert.strictEqual(tokenizer.takeLastTokens(source, 28).tokens.length, 28); + assert.strictEqual(tokenizer.takeLastTokens(source, 5).tokens.length, 5); + assert.strictEqual(tokenizer.takeLastTokens(source, 0).tokens.length, 0); + assert.strictEqual(tokenizer.takeLastTokens(source, 1).tokens.length, 1); + assert.strictEqual(tokenizer.takeLastTokens(source, 1000).tokens.length, 1000); + assert.strictEqual(tokenizer.takeLastTokens(source, 100000).text, source); + assert.strictEqual(tokenizer.takeLastTokens('\n\n\n', 1).tokens.length, 1); + }); + + test('takeLastTokens returns a suffix of the sought length', function () { + function check(n: number): void { + const { text: suffix } = tokenizer.takeLastTokens(source, n); + assert.strictEqual(tokenizer.tokenLength(suffix), n); + assert.strictEqual(suffix, source.substring(source.length - suffix.length)); + } + check(0); + check(1); + check(5); + check(29); + check(30); + check(100); + check(1000); + assert.strictEqual(tokenizer.takeLastTokens(source, 100000).text, source); + }); + + test('test takeLastLinesTokens', function () { + let example = 'a b c\nd e f\ng h i'; + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 3), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 4), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 5), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 6), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 7), 'd e f\ng h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 11), example); + example = 'a b\n\n c d'; + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 2), ' c d'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 3), '\n c d'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 4), '\n c d'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 5), 'a b\n\n c d'); + }); + + test('takeFirstTokens return corresponding text and tokens', function () { + let prefix = tokenizer.takeFirstTokens(source, 30); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + prefix = tokenizer.takeFirstTokens(source, 0); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + prefix = tokenizer.takeFirstTokens('', 30); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + prefix = tokenizer.takeFirstTokens('', 0); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + }); + + test('takeFirstTokens invariant of ending position', function () { + const prefix = tokenizer.takeFirstTokens(source, 29).text; + const expected = `""" +This is an example Python source file to use as test data. It's pulled from the synth repo +with minor edits to make it`; + assert.strictEqual(prefix, expected); + assert.strictEqual(tokenizer.tokenLength(prefix), 29); + assert.strictEqual(prefix, tokenizer.takeFirstTokens(source.substring(0, 150), 29).text); + assert.strictEqual(prefix, tokenizer.takeFirstTokens(source.substring(0, 200), 29).text); + }); + + test('takeFirstTokens returns the desired number of tokens', function () { + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 30).text), 30); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 29).text), 29); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 28).text), 28); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 5).text), 5); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 0).text), 0); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 1).text), 1); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 1000).text), 1000); + assert.strictEqual(tokenizer.takeFirstTokens(source, 100000).text, source); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens('\n\n\n', 1).text), 1); + }); + + test('takeFirstTokens returns a prefix of the sought length', function () { + function check(n: number): void { + const prefix = tokenizer.takeFirstTokens(source, n).text; + assert.strictEqual(tokenizer.tokenLength(prefix), n); + assert.strictEqual(prefix, source.substring(0, prefix.length)); + } + check(0); + check(1); + check(5); + check(29); + check(30); + check(100); + check(1000); + assert.strictEqual(tokenizer.takeFirstTokens(source, 100000).text, source); + }); + + /** + * Long sequences of spaces are tokenized as a sequence of 16-space tokens. This tests that + * the logic in takeFirstTokens correctly handles very long tokens. + */ + test('takeFirstTokens handles very long tokens', function () { + this.timeout(15000); + const longestSpaceToken = ' '.repeat(4000); + const tokens = tokenizer.takeFirstTokens(longestSpaceToken, 30); + assert.strictEqual(tokenizer.tokenLength(tokens.text), 30); + }); +}); + +suite('Tokenizer Test Suite - o200k', function () { + const tokenizer = getTokenizer(TokenizerName.o200k); + + test('empty string', function () { + const str = ''; + assert.deepStrictEqual(tokenizer.tokenize(str), []); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('space', function () { + const str = ' '; + assert.deepStrictEqual(tokenizer.tokenize(str), [220]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('tab', function () { + const str = '\t'; + assert.deepStrictEqual(tokenizer.tokenize(str), [197]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('simple text', function () { + const str = 'This is some text'; + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + assert.deepStrictEqual(tokenizer.tokenize(str), [2500, 382, 1236, 2201]); + }); + + test('multi-token word', function () { + const str = 'indivisible'; + assert.deepStrictEqual(tokenizer.tokenize(str), [521, 349, 181386]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('emojis', function () { + const str = 'hello ๐Ÿ‘‹ world ๐ŸŒ'; + assert.deepStrictEqual(tokenizer.tokenize(str), [24912, 61138, 233, 2375, 130321, 235]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('contractions', function () { + const str = `you'll`; + assert.deepStrictEqual(tokenizer.tokenize(str), [13320, 6090]); + assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); + }); + + test('assert that consecutive newline is never tokenized as multiple newlines', function () { + // This is due to a regular expression change in the tokenizer. + + // Loop through all possible ascii numbers and letters + for (let i = 0; i < 128; i++) { + const char = String.fromCharCode(i); + if (char !== '\n') { + assert.deepStrictEqual(tokenizer.tokenLength(`\n\n${char}`), 2); + } + } + + // Test special characters + assert.deepStrictEqual(tokenizer.tokenize('\n\n๐Ÿ‘‹'), [279, 28823, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n '), [279, 220]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n ๐Ÿ‘‹'), [279, 61138, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n\t'), [279, 197]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n\r'), [279, 201]); + + // New lines are treated specially tho + for (let i = 1; i < 10; i++) { + assert.deepStrictEqual(tokenizer.tokenLength('\n'.repeat(i)), 1); + } + }); + + test('tokenizeStrings', function () { + const tokens_s = tokenizer.tokenizeStrings(source); + assert.strictEqual(tokens_s.join(''), source, 'tokenizeStrings does not join to form the input string'); + const tokens = tokenizer.tokenize(source); + assert.strictEqual(tokens_s.length, tokens.length, 'tokenizeStrings should have same length as tokenize'); + const half = Math.floor(tokens_s.length / 2); + assert.strictEqual( + tokens_s.slice(0, half).join(''), + tokenizer.detokenize(tokens.slice(0, half)), + 'tokenizeStrings slice should represent the corresponding slice with tokenize' + ); + }); + + test('takeLastTokens invariant of starting position', function () { + const suffix = tokenizer.takeLastTokens(source, 25); + assert.strictEqual( + suffix.text, + `To the import {imp.module} as {imp.as_name}, we add the following comment: {description}")\n return description` + ); + assert.strictEqual(suffix.tokens.length, 25); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(50), 25).text); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(100), 25).text); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(150), 25).text); + assert.strictEqual(suffix.text, tokenizer.takeLastTokens(source.substring(200), 25).text); + }); + + test('takeLastTokens returns the desired number of tokens', function () { + assert.strictEqual(tokenizer.takeLastTokens(source, 30).tokens.length, 30); + assert.strictEqual(tokenizer.takeLastTokens(source, 29).tokens.length, 29); + assert.strictEqual(tokenizer.takeLastTokens(source, 28).tokens.length, 28); + assert.strictEqual(tokenizer.takeLastTokens(source, 5).tokens.length, 5); + assert.strictEqual(tokenizer.takeLastTokens(source, 0).tokens.length, 0); + assert.strictEqual(tokenizer.takeLastTokens(source, 1).tokens.length, 1); + assert.strictEqual(tokenizer.takeLastTokens(source, 1000).tokens.length, 1000); + assert.strictEqual(tokenizer.takeLastTokens(source, 100000).text, source); + assert.strictEqual(tokenizer.takeLastTokens('\n\n\n', 1).tokens.length, 1); + }); + + test('takeLastTokens returns a suffix of the sought length', function () { + function check(n: number): void { + const { text: suffix } = tokenizer.takeLastTokens(source, n); + assert.strictEqual(tokenizer.tokenLength(suffix), n); + assert.strictEqual(suffix, source.substring(source.length - suffix.length)); + } + check(0); + check(1); + check(5); + check(29); + check(30); + check(100); + check(1000); + assert.strictEqual(tokenizer.takeLastTokens(source, 100000).text, source); + }); + + test('test takeLastLinesTokens', function () { + let example = 'a b c\nd e f\ng h i'; + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 3), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 4), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 5), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 6), 'g h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 7), 'd e f\ng h i'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 11), example); + example = 'a b\n\n c d'; + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 2), ' c d'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 3), '\n c d'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 4), '\n c d'); + assert.strictEqual(tokenizer.takeLastLinesTokens(example, 5), 'a b\n\n c d'); + }); + + test('takeFirstTokens return corresponding text and tokens', function () { + let prefix = tokenizer.takeFirstTokens(source, 30); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + prefix = tokenizer.takeFirstTokens(source, 0); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + prefix = tokenizer.takeFirstTokens('', 30); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + prefix = tokenizer.takeFirstTokens('', 0); + assert.strictEqual(prefix.text, tokenizer.detokenize(prefix.tokens)); + }); + + test('takeFirstTokens invariant of ending position', function () { + const prefix = tokenizer.takeFirstTokens(source, 29).text; + const expected = `""" +This is an example Python source file to use as test data. It's pulled from the synth repo +with minor edits to make it a`; + assert.strictEqual(prefix, expected); + assert.strictEqual(tokenizer.tokenLength(prefix), 29); + assert.strictEqual(prefix, tokenizer.takeFirstTokens(source.substring(0, 150), 29).text); + assert.strictEqual(prefix, tokenizer.takeFirstTokens(source.substring(0, 200), 29).text); + }); + + test('takeFirstTokens returns the desired number of tokens', function () { + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 30).text), 30); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 29).text), 29); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 28).text), 28); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 5).text), 5); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 0).text), 0); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 1).text), 1); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens(source, 1000).text), 1000); + assert.strictEqual(tokenizer.takeFirstTokens(source, 100000).text, source); + assert.strictEqual(tokenizer.tokenLength(tokenizer.takeFirstTokens('\n\n\n', 1).text), 1); + }); + + test('takeFirstTokens returns a prefix of the sought length', function () { + function check(n: number): void { + const prefix = tokenizer.takeFirstTokens(source, n).text; + assert.strictEqual(tokenizer.tokenLength(prefix), n); + assert.strictEqual(prefix, source.substring(0, prefix.length)); + } + check(0); + check(1); + check(5); + check(29); + check(30); + check(100); + check(1000); + assert.strictEqual(tokenizer.takeFirstTokens(source, 100000).text, source); + }); + + /** + * Long sequences of spaces are tokenized as a sequence of 16-space tokens. This tests that + * the logic in takeFirstTokens correctly handles very long tokens. + */ + test('takeFirstTokens handles very long tokens', function () { + this.timeout(15000); + const longestSpaceToken = ' '.repeat(4000); + const tokens = tokenizer.takeFirstTokens(longestSpaceToken, 30); + assert.strictEqual(tokenizer.tokenLength(tokens.text), 30); + }); +}); + +suite('ApproximateTokenizer', function () { + const cl100kTokenizer = new ApproximateTokenizer(TokenizerName.cl100k, 'python'); + const o200kTokenizer = new ApproximateTokenizer(TokenizerName.o200k, 'python'); + const defaultTokenizer = new ApproximateTokenizer(); // o200k, no language; + + suite('tokenizeStrings', function () { + test('should split text into chunks of 4 characters', function () { + const result = defaultTokenizer.tokenizeStrings('abcdefgh'); + assert.deepStrictEqual(result, ['abcd', 'efgh']); + }); + + test('should handle text not divisible by 4', function () { + const result = defaultTokenizer.tokenizeStrings('abcdefg'); + assert.deepStrictEqual(result, ['abcd', 'efg']); + }); + + test('should handle empty string', function () { + const result = defaultTokenizer.tokenizeStrings(''); + assert.deepStrictEqual(result, []); + }); + + test('should handle single character', function () { + const result = defaultTokenizer.tokenizeStrings('a'); + assert.deepStrictEqual(result, ['a']); + }); + }); + + suite('tokenize', function () { + test('should convert string chunks to numeric tokens', function () { + const result = defaultTokenizer.tokenize('ab'); + assert.ok(Array.isArray(result)); + assert.strictEqual(result.length, 1); + assert.strictEqual(typeof result[0], 'number'); + }); + + test('should produce consistent tokens for same input', function () { + const result1 = defaultTokenizer.tokenize('test'); + const result2 = defaultTokenizer.tokenize('test'); + assert.deepStrictEqual(result1, result2); + }); + }); + + suite('detokenize', function () { + test('should convert tokens back to string', function () { + const original = 'test'; + const tokens = defaultTokenizer.tokenize(original); + const result = defaultTokenizer.detokenize(tokens); + assert.strictEqual(result, original); + }); + + test('should handle empty token array', function () { + const result = defaultTokenizer.detokenize([]); + assert.strictEqual(result, ''); + }); + }); + + test('tokenLength', function () { + assert.strictEqual(cl100kTokenizer.tokenLength('a b c'), 2); + }); + + test('tokenLength with language take approximated char chunks', function () { + assert.strictEqual(cl100kTokenizer.tokenLength('abc def gh'), 3); + }); + + test('tokenLength with no language take 4 char chunks', function () { + const str = 'w'.repeat(400); + assert.strictEqual(cl100kTokenizer.tokenLength(str), 101); + assert.strictEqual(defaultTokenizer.tokenLength(str), 100); + }); + + test('tokenLength approximated char chunks are correct for each approximated tokenizer', function () { + const str = 'w'.repeat(400); + assert.strictEqual(cl100kTokenizer.tokenLength(str), 101); + assert.strictEqual(o200kTokenizer.tokenLength(str), 99); + }); + + test('takeFirstTokens', function () { + const first2Tokens = cl100kTokenizer.takeFirstTokens('123 456 7890', 2); + assert.deepStrictEqual(first2Tokens, { + text: '123 456', + tokens: [0, 1], + }); + assert.deepStrictEqual(cl100kTokenizer.tokenLength(first2Tokens.text), 2); + }); + + test('takeFirstTokens returns the full string if shorter', function () { + const first100Tokens = cl100kTokenizer.takeFirstTokens('123 456 7890', 100); + assert.deepStrictEqual(first100Tokens, { + text: '123 456 7890', + tokens: [0, 1, 2, 3], + }); + assert.deepStrictEqual(cl100kTokenizer.tokenLength(first100Tokens.text), 4); + }); + + test('takeLastTokens', function () { + const last2Tokens = cl100kTokenizer.takeLastTokens('123 456 7890', 2); + assert.deepStrictEqual(last2Tokens, { + text: '56 7890', + tokens: [0, 1], + }); + assert.deepStrictEqual(cl100kTokenizer.tokenLength(last2Tokens.text), 2); + }); + + test('takeLastTokens returns the full string if shorter', function () { + const last100Tokens = cl100kTokenizer.takeLastTokens('123 456 7890', 100); + assert.deepStrictEqual(last100Tokens, { + text: '123 456 7890', + tokens: [0, 1, 2, 3], + }); + assert.deepStrictEqual(cl100kTokenizer.tokenLength(last100Tokens.text), 4); + }); + + suite('takeLastLinesTokens', function () { + test('should return complete lines from suffix', function () { + const text = 'line1\nline2\nline3\nline4'; + const result = cl100kTokenizer.takeLastLinesTokens(text, 4); + assert.strictEqual(result, 'line3\nline4'); + }); + + test('should handle text already within token limit', function () { + const text = 'short\ntext'; + const result = cl100kTokenizer.takeLastLinesTokens(text, 100); + assert.strictEqual(result, text); + }); + + test('should handle text ending with newline', function () { + const text = 'line1\nline2\n'; + const result = cl100kTokenizer.takeLastLinesTokens(text, 10); + assert.strictEqual(typeof result, 'string'); + }); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/test/windowDelineation.test.ts b/completions-sample-code/vscode-node/prompt/src/test/windowDelineation.test.ts new file mode 100644 index 0000000..28ac3a4 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/test/windowDelineation.test.ts @@ -0,0 +1,106 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { getIndentationWindowsDelineations } from '../snippetInclusion/windowDelineations'; +import * as assert from 'assert'; +import dedent from 'ts-dedent'; + +const SOURCE = { + source: dedent` + f1: + a1 + f2: + a2 + a3 +`, + name: '', +}; + +suite('Test window delineation', function () { + test('Correct line number range, standard input', function () { + const testLineNumbers: [number, number][] = getIndentationWindowsDelineations( + SOURCE.source.split('\n'), + 'python', + 1, + 3 + ); + const correctLineNumbers: [number, number][] = [ + [0, 2], // f1: a1 + [1, 2], // a1 + [2, 5], // f2: a2 a3 + [3, 4], // a2 + [4, 5], // a3 + ]; + assert.deepStrictEqual(testLineNumbers.sort(), correctLineNumbers.sort()); + }); + test('Correct line number range, standard input, decreased maxLength', function () { + const testLineNumbers: [number, number][] = getIndentationWindowsDelineations( + SOURCE.source.split('\n'), + 'python', + 1, + 2 + ); + const correctLineNumbers: [number, number][] = [ + [0, 2], // f1: a1 + [1, 2], // a1 + [3, 4], // a2 + [4, 5], // a3 + // We lose [2, 5] f2: a2 a3 as too long + // But we gain the following which were previously swallowed up by [2, 5] + [2, 4], // f2: a2 + [3, 5], // a2 a3 + ]; + assert.deepStrictEqual(testLineNumbers.sort(), correctLineNumbers.sort()); + }); + test('Correct line number range, standard input, increased minLength', function () { + const testLineNumbers: [number, number][] = getIndentationWindowsDelineations( + SOURCE.source.split('\n'), + 'python', + 2, + 3 + ); + const correctLineNumbers: [number, number][] = [ + [0, 2], // f1: a1 + [2, 5], // f2: a2 a3 + // We lose the following as too short + // [1, 2] a1 + // [3, 4] a2 + // [4, 5] a3 + ]; + assert.deepStrictEqual(testLineNumbers.sort(), correctLineNumbers.sort()); + }); + + test('Correct line number range, flat input', function () { + const source: string = dedent` + a1 + a2 + a3 + `; + const testLineNumbers: [number, number][] = getIndentationWindowsDelineations( + source.split('\n'), + 'python', + 1, + 3 + ); + const correctLineNumbers: [number, number][] = [ + [0, 1], // a1 + [1, 2], // a2 + [2, 3], // a3 + [0, 3], // a1 a2 a3 + // Don't get [0, 2] nor [1, 3] because they not single children nor the whole tree + ]; + assert.deepStrictEqual(testLineNumbers.sort(), correctLineNumbers.sort()); + }); + + test('Check degenerate case', function () { + const testLineNumbers: [number, number][] = getIndentationWindowsDelineations( + SOURCE.source.split('\n'), + 'python', + 0, + 0 + ); + const correctLineNumbers: [number, number][] = []; + assert.deepStrictEqual(testLineNumbers.sort(), correctLineNumbers.sort()); + }); +}); diff --git a/completions-sample-code/vscode-node/prompt/src/tokenization/index.ts b/completions-sample-code/vscode-node/prompt/src/tokenization/index.ts new file mode 100644 index 0000000..33df758 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/tokenization/index.ts @@ -0,0 +1,6 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export * from './tokenizer'; diff --git a/completions-sample-code/vscode-node/prompt/src/tokenization/tokenizer.ts b/completions-sample-code/vscode-node/prompt/src/tokenization/tokenizer.ts new file mode 100644 index 0000000..c6db509 --- /dev/null +++ b/completions-sample-code/vscode-node/prompt/src/tokenization/tokenizer.ts @@ -0,0 +1,385 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TikTokenizer, createTokenizer, getRegexByEncoder, getSpecialTokensByEncoder } from '@microsoft/tiktokenizer'; +import { parseTikTokenBinary } from '../../../../../../platform/tokenizer/node/parseTikTokens'; +import { CopilotPromptLoadFailure } from '../error'; +import { locateFile } from '../fileLoader'; + +export enum TokenizerName { + cl100k = 'cl100k_base', + o200k = 'o200k_base', + mock = 'mock', +} + +const tokenizers = new Map<TokenizerName, Tokenizer>(); + +export function getTokenizer(name: TokenizerName = TokenizerName.o200k): Tokenizer { + let tokenizer = tokenizers.get(name); + if (tokenizer !== undefined) { return tokenizer; } + // Fallback to o200k + tokenizer = tokenizers.get(TokenizerName.o200k); + if (tokenizer !== undefined) { return tokenizer; } + // Fallback to approximate tokenizer + return new ApproximateTokenizer(); +} + +export interface Tokenizer { + /** + * Return the length of `text` in number of tokens. + * + * @param text - The input text + * @returns + */ + tokenLength(text: string): number; + + /** + * Returns the tokens created from tokenizing `text`. + * @param text The text to tokenize + */ + tokenize(text: string): number[]; + + /** + * Returns the string representation of the tokens in `tokens`, given in integer + * representation. + * + * This is the functional inverse of `tokenize`. + */ + detokenize(tokens: number[]): string; + + /** + * Returns the tokenization of the input string as a list of strings. + * + * The concatenation of the output of this function is equal to the input. + */ + tokenizeStrings(text: string): string[]; + + /** + * Return a suffix of `text` which is `n` tokens long. + * If `text` is at most `n` tokens, return `text`. + * + * Note: This implementation does not attempt to return + * the longest possible suffix, only *some* suffix of at + * most `n` tokens. + * + * @param text - The text from which to take + * @param n - How many tokens to take + * @returns A suffix of `text`, as a `{ text: string, tokens: number[] }`. + */ + takeLastTokens(text: string, n: number): { text: string; tokens: number[] }; + + /** + * Return a prefix of `text` which is `n` tokens long. + * If `text` is at most `n` tokens, return `text`. + * + * Note: This implementation does not attempt to return + * the longest possible prefix, only *some* prefix of at + * most `n` tokens. + * + * @param text - The text from which to take + * @param n - How many tokens to take + * @returns A prefix of `text`, as a `{ text: string, tokens: number[] }`. + */ + takeFirstTokens(text: string, n: number): { text: string; tokens: number[] }; + + /** + * Return the longest suffix of `text` of complete lines and is at most + * `n` tokens long. + * @param text - The text from which to take + * @param n - How many tokens to take + */ + takeLastLinesTokens(text: string, n: number): string; +} + +export class TTokenizer implements Tokenizer { + constructor(private readonly _tokenizer: TikTokenizer) { } + + static async create(encoder: TokenizerName): Promise<TTokenizer> { + try { + const tokenizer = createTokenizer( + parseTikTokenBinary(locateFile(`${encoder}.tiktoken`)), + getSpecialTokensByEncoder(encoder), + getRegexByEncoder(encoder), + 32768 + ); + return new TTokenizer(tokenizer); + } catch (e: unknown) { + if (e instanceof Error) { + throw new CopilotPromptLoadFailure(`Could not load tokenizer`, e); + } + throw e; + } + } + + tokenize(text: string): number[] { + return this._tokenizer.encode(text); + } + + detokenize(tokens: number[]): string { + return this._tokenizer.decode(tokens); + } + + tokenLength(text: string): number { + return this.tokenize(text).length; + } + + tokenizeStrings(text: string): string[] { + const tokens = this.tokenize(text); + return tokens.map(token => this.detokenize([token])); + } + + takeLastTokens(text: string, n: number): { text: string; tokens: number[] } { + if (n <= 0) { return { text: '', tokens: [] }; } + + // Find long enough suffix of text that has >= n + 2 tokens + // We add the 2 extra tokens to avoid the edge case where + // we cut at exactly n tokens and may get an odd tokenization. + const CHARS_PER_TOKENS_START = 4; + const CHARS_PER_TOKENS_ADD = 1; + let chars = Math.min(text.length, n * CHARS_PER_TOKENS_START); //First guess + let suffix = text.slice(-chars); + let suffixT = this.tokenize(suffix); + while (suffixT.length < n + 2 && chars < text.length) { + chars = Math.min(text.length, chars + n * CHARS_PER_TOKENS_ADD); + suffix = text.slice(-chars); + suffixT = this.tokenize(suffix); + } + if (suffixT.length < n) { + // text must be <= n tokens long + return { text, tokens: suffixT }; + } + // Return last n tokens + suffixT = suffixT.slice(-n); + return { text: this.detokenize(suffixT), tokens: suffixT }; + } + + takeFirstTokens(text: string, n: number): { text: string; tokens: number[] } { + if (n <= 0) { return { text: '', tokens: [] }; } + + // Find long enough suffix of text that has >= n + 2 tokens + // We add the 2 extra tokens to avoid the edge case where + // we cut at exactly n tokens and may get an odd tokenization. + const CHARS_PER_TOKENS_START = 4; + const CHARS_PER_TOKENS_ADD = 1; + let chars = Math.min(text.length, n * CHARS_PER_TOKENS_START); //First guess + let prefix = text.slice(0, chars); + let prefix_t = this.tokenize(prefix); + while (prefix_t.length < n + 2 && chars < text.length) { + chars = Math.min(text.length, chars + n * CHARS_PER_TOKENS_ADD); + prefix = text.slice(0, chars); + prefix_t = this.tokenize(prefix); + } + if (prefix_t.length < n) { + // text must be <= n tokens long + return { + text: text, + tokens: prefix_t, + }; + } + // Return first n tokens + // This implicit "truncate final tokens" text processing algorithm + // could be extracted into a generic snippet text processing function managed by the SnippetTextProcessor class. + prefix_t = prefix_t.slice(0, n); + return { + text: this.detokenize(prefix_t), + tokens: prefix_t, + }; + } + + takeLastLinesTokens(text: string, n: number): string { + const { text: suffix } = this.takeLastTokens(text, n); + if (suffix.length === text.length || text[text.length - suffix.length - 1] === '\n') { + // Edge case: We already took whole lines + return suffix; + } + const newline = suffix.indexOf('\n'); + return suffix.substring(newline + 1); + } +} + +class MockTokenizer implements Tokenizer { + private hash = (str: string) => { + let hash = 0; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = (hash << 5) - hash + char; + hash &= hash & 0xffff; + } + return hash; + }; + + tokenize(text: string): number[] { + return this.tokenizeStrings(text).map(this.hash); + } + detokenize(tokens: number[]): string { + // Note because this is using hashing to mock tokenization, it is not + // reversible, so detokenize will not return the original input. + return tokens.map(token => token.toString()).join(' '); + } + tokenizeStrings(text: string): string[] { + return text.split(/\b/); + } + tokenLength(text: string): number { + return this.tokenizeStrings(text).length; + } + + takeLastTokens(text: string, n: number): { text: string; tokens: number[] } { + const tokens = this.tokenizeStrings(text).slice(-n); + return { text: tokens.join(''), tokens: tokens.map(this.hash) }; + } + takeFirstTokens(text: string, n: number): { text: string; tokens: number[] } { + const tokens = this.tokenizeStrings(text).slice(0, n); + return { text: tokens.join(''), tokens: tokens.map(this.hash) }; + } + takeLastLinesTokens(text: string, n: number): string { + const { text: suffix } = this.takeLastTokens(text, n); + if (suffix.length === text.length || text[text.length - suffix.length - 1] === '\n') { + // Edge case: We already took whole lines + return suffix; + } + const newline = suffix.indexOf('\n'); + return suffix.substring(newline + 1); + } +} + +// These are the effective token lengths for each language. They are based on empirical data to balance the risk of accidental overflow and overeager elision. +// Note: These may need to be recalculated in the future if typical prompt lengths are significantly changed. +const EFFECTIVE_TOKEN_LENGTH: Partial<Record<TokenizerName, Record<string, number>>> = { + [TokenizerName.cl100k]: { + python: 3.99, + typescript: 4.54, + typescriptreact: 4.58, + javascript: 4.76, + csharp: 5.13, + java: 4.86, + cpp: 3.85, + php: 4.1, + html: 4.57, + vue: 4.22, + go: 3.93, + dart: 5.66, + javascriptreact: 4.81, + css: 3.37, + }, + [TokenizerName.o200k]: { + python: 4.05, + typescript: 4.12, + typescriptreact: 5.01, + javascript: 4.47, + csharp: 5.47, + java: 4.86, + cpp: 3.8, + php: 4.35, + html: 4.86, + vue: 4.3, + go: 4.21, + dart: 5.7, + javascriptreact: 4.83, + css: 3.33, + }, +}; + +/** Max decimals per code point for ApproximateTokenizer mock tokenization. */ +const MAX_CODE_POINT_SIZE = 4; + +/** A best effort tokenizer computing the length of the text by dividing the + * number of characters by estimated constants near the number 4. + * It is not a real tokenizer. */ +export class ApproximateTokenizer implements Tokenizer { + tokenizerName: TokenizerName; + + constructor( + tokenizerName: TokenizerName = TokenizerName.o200k, + private languageId?: string + ) { + this.tokenizerName = tokenizerName; + } + + tokenize(text: string): number[] { + return this.tokenizeStrings(text).map(substring => { + let charCode = 0; + for (let i = 0; i < substring.length; i++) { + charCode = charCode * Math.pow(10, MAX_CODE_POINT_SIZE) + substring.charCodeAt(i); + } + return charCode; + }); + } + + detokenize(tokens: number[]): string { + return tokens + .map(token => { + const chars = []; + let charCodes = token.toString(); + while (charCodes.length > 0) { + const charCode = charCodes.slice(-MAX_CODE_POINT_SIZE); + const char = String.fromCharCode(parseInt(charCode)); + chars.unshift(char); + charCodes = charCodes.slice(0, -MAX_CODE_POINT_SIZE); + } + return chars.join(''); + }) + .join(''); + } + + tokenizeStrings(text: string): string[] { + // Mock tokenize by defaultETL + return text.match(/.{1,4}/g) ?? []; + } + + private getEffectiveTokenLength(): number { + // Our default is 4, used for tail languages and error handling + const defaultETL = 4; + + if (this.tokenizerName && this.languageId) { + // Use our calculated effective token length for head languages + return EFFECTIVE_TOKEN_LENGTH[this.tokenizerName]?.[this.languageId] ?? defaultETL; + } + + return defaultETL; + } + + tokenLength(text: string): number { + return Math.ceil(text.length / this.getEffectiveTokenLength()); + } + + takeLastTokens(text: string, n: number): { text: string; tokens: number[] } { + if (n <= 0) { return { text: '', tokens: [] }; } + // Return the last characters approximately. It doesn't matter what we return as token, just that it has the correct length. + const suffix = text.slice(-Math.floor(n * this.getEffectiveTokenLength())); + return { text: suffix, tokens: Array.from({ length: this.tokenLength(suffix) }, (_, i) => i) }; + } + + takeFirstTokens(text: string, n: number): { text: string; tokens: number[] } { + if (n <= 0) { return { text: '', tokens: [] }; } + // Return the first characters approximately. + const prefix = text.slice(0, Math.floor(n * this.getEffectiveTokenLength())); + return { text: prefix, tokens: Array.from({ length: this.tokenLength(prefix) }, (_, i) => i) }; + } + + takeLastLinesTokens(text: string, n: number): string { + const { text: suffix } = this.takeLastTokens(text, n); + if (suffix.length === text.length || text[text.length - suffix.length - 1] === '\n') { + // Edge case: We already took whole lines + return suffix; + } + const newline = suffix.indexOf('\n'); + return suffix.substring(newline + 1); + } +} + +async function setTokenizer(name: TokenizerName) { + try { + const tokenizer = await TTokenizer.create(name); + tokenizers.set(name, tokenizer); + } catch { + // Ignore errors loading tokenizer + } +} + +/** Load tokenizers on start. Export promise for to be awaited by initialization. */ +export const initializeTokenizers = (async () => { + tokenizers.set(TokenizerName.mock, new MockTokenizer()); + await Promise.all([setTokenizer(TokenizerName.cl100k), setTokenizer(TokenizerName.o200k)]); +})(); diff --git a/completions-sample-code/vscode-node/types/src/auth.ts b/completions-sample-code/vscode-node/types/src/auth.ts new file mode 100644 index 0000000..63123de --- /dev/null +++ b/completions-sample-code/vscode-node/types/src/auth.ts @@ -0,0 +1,23 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Static, Type } from '@sinclair/typebox'; +import * as lsp from 'vscode-languageserver-protocol'; + +export const DidChangeAuthParams = Type.Object({ + accessToken: Type.Optional(Type.String({ minLength: 1 })), + handle: Type.Optional(Type.String({ minLength: 1 })), + login: Type.Optional(Type.String({ minLength: 1 })), + githubAppId: Type.Optional(Type.String({ minLength: 1 })), + apiUrl: Type.Optional(Type.String({})), + serverUrl: Type.Optional(Type.String({})), + tokenEndpoint: Type.Optional(Type.String({})), +}); +export type DidChangeAuthParams = Static<typeof DidChangeAuthParams>; + +export namespace DidChangeAuthNotification { + export const method = 'github/didChangeAuth'; + export const type = new lsp.ProtocolNotificationType<DidChangeAuthParams, void>(method); +} diff --git a/completions-sample-code/vscode-node/types/src/codeCitation.ts b/completions-sample-code/vscode-node/types/src/codeCitation.ts new file mode 100644 index 0000000..9982939 --- /dev/null +++ b/completions-sample-code/vscode-node/types/src/codeCitation.ts @@ -0,0 +1,26 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Static } from '@sinclair/typebox'; +import * as lsp from 'vscode-languageserver-protocol'; +import { RangeSchema } from './core'; + +type IPCodeCitation = { + license: string; + url: string; +}; + +type CopilotIPCodeCitationNotificationParams = { + uri: string; + version: number; + range: Static<typeof RangeSchema>; + matchingText: string; + citations: IPCodeCitation[]; +}; + +export namespace CopilotIPCodeCitationNotification { + export const method = 'copilot/ipCodeCitation'; + export const type = new lsp.NotificationType<CopilotIPCodeCitationNotificationParams>(method); +} diff --git a/completions-sample-code/vscode-node/types/src/contextProviderApiV1.ts b/completions-sample-code/vscode-node/types/src/contextProviderApiV1.ts new file mode 100644 index 0000000..254977f --- /dev/null +++ b/completions-sample-code/vscode-node/types/src/contextProviderApiV1.ts @@ -0,0 +1,212 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { Diagnostic, Uri } from 'vscode'; +import { + CancellationToken, + Disposable, + DocumentSelector, + DocumentUri, + Position, + TextEdit, +} from 'vscode-languageserver-protocol'; + +/** + * The ContextProvider API allows extensions to provide additional context items that + * Copilot can use in its prompt. This file contains type definitions for the methods + * and the data structures used by the API. + * + * Note: providing context is not enough to ensure that the context will be used in the prompt. + * + * The API is exposed as an export of the Copilot extension. To use it, you can cast the + * exported object to the ContextProviderApiV1 interface. + * + * Example: + * ``` + * const copilot = vscode.extensions.getExtension("github.copilot"); + * const contextProviderAPI = copilot.exports.getContextProviderAPI("v1") as ContextProviderApiV1; + * ``` + */ +export interface ContextProviderApiV1 { + registerContextProvider<T extends SupportedContextItem>(provider: ContextProvider<T>): Disposable; +} + +/** + * Each extension can register a number of context providers, uniquely identified by their ID. + * In addition, each provider has to provide: + * - a DocumentSelector, to specify the file types for which the provider is active + * - a ContextResolver, a function that returns the context items for a given request + * + * Example: + * ``` + * contextProviderAPI.registerContextProvider<Trait>({ + * id: "pythonProvider", + * selector: [{ language: "python" }], + * resolver: { + * resolve: async (request, token) => { + * return [{name: 'traitName', value: 'traitValue'}]; + * } + * } + * }); + * ``` + */ +export interface ContextProvider<T extends SupportedContextItem> { + id: string; + selector: DocumentSelector; + resolver: ContextResolver<T>; +} + +export type ResolveOnTimeoutResult<T> = T | readonly T[]; +export type ResolveResult<T> = Promise<T> | Promise<readonly T[]> | AsyncIterable<T>; + +export interface ContextResolver<T extends SupportedContextItem> { + resolve(request: ResolveRequest, token: CancellationToken): ResolveResult<T>; + // Optional method to be invoked if the request timed out. This requests additional context items. + resolveOnTimeout?(request: ResolveRequest): ResolveOnTimeoutResult<T> | undefined; +} + +/** + * The first argument of the resolve method is a ResolveRequest object, which informs + * the provider about: + * - the completionId, a unique identifier for the completion request + * - the documentContext, which contains information about the document for which the context is requested + * - the activeExperiments, a map of active experiments and their values + * - the timeBudget the provider has to provide context items + * - the previousUsageStatistics, which contains information about the last request to the provider + */ +export type ResolutionStatus = 'full' | 'partial' | 'none' | 'error'; +export type UsageStatus = ResolutionStatus | 'partial_content_excluded' | 'none_content_excluded'; + +export type ContextItemUsageDetails = { + id: string; + type: SupportedContextItemType; + origin?: ContextItemOrigin; +} & ( + | { + usage: Extract<UsageStatus, 'full' | 'partial' | 'partial_content_excluded'>; + expectedTokens: number; + actualTokens: number; + } + | { usage: Extract<UsageStatus, 'none' | 'none_content_excluded' | 'error'> } + ); + +export type ContextUsageStatistics = { + usage: UsageStatus; + resolution: ResolutionStatus; + usageDetails?: ContextItemUsageDetails[]; +}; + +export type ProposedTextEdit = TextEdit & { + positionAfterEdit: Position; + // Indicates whether the edit is suggested by the IDE. Otherwise it's assumed to be speculative + source?: 'selectedCompletionInfo'; +}; + +export interface DocumentContext { + uri: DocumentUri; + languageId: string; + version: number; + // Position and offset are relative to the provided version of the document. + // The position after an edit is applied is found in ProposedTextEdit.positionAfterEdit. + /** + * @deprecated Use `position` instead. + */ + offset: number; + position: Position; + proposedEdits?: ProposedTextEdit[]; +} +export interface ResolveRequest { + // A unique ID to correlate the request with the completion request. + completionId: string; + opportunityId: string; + documentContext: DocumentContext; + + activeExperiments: Map<string, string | number | boolean | string[]>; + + /** + * The number of milliseconds for the context provider to provide context items. + * After the time budget runs out, the request will be cancelled via the CancellationToken. + * Providers can use this value as a hint when computing context. Providers should expect the + * request to be cancelled once the time budget runs out. + * + * @deprecated Use `timeoutEnd` instead. + */ + timeBudget: number; + + /** + * Unix timestamp representing the exact time the request will be cancelled via the CancellationToken. + */ + timeoutEnd: number; + + /** + * Various statistics about the last completion request. This can be used by the context provider + * to make decisions about what context to provide for the current call. + */ + previousUsageStatistics?: ContextUsageStatistics; + + /** + * Data from completionItem + * + * See https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#completionItem + */ + data?: unknown; +} + +/** + * These are the data types that can be provided by a context provider. Any non-conforming + * context items will be filtered out. + */ +interface ContextItem { + /** + * Specifies the relative importance with respect to items of the same type. + * Cross-type comparisons is currently handled by the wishlist. + * Accepted values are integers in the range [0, 100], where 100 is the highest importance. + * Items with non-conforming importance values will be filtered out. + * Default value is 0. + */ + importance?: number; + + /** + * A unique ID for the context item, used to provide detailed statistics about + * the item's usage. If an ID is not provided, it will be generated randomly. + */ + id?: string; + + /** + * Specifies where the context item comes from, mostly relevant for LSP providers. + * - request: context is provided in the completion request + * - update: context is provided via context/update + */ + origin?: ContextItemOrigin; +} + +// A key-value pair used for short string snippets. +export interface Trait extends ContextItem { + name: string; + value: string; +} + +// Code snippet extracted from a file. The URI is used for content exclusion. +export interface CodeSnippet extends ContextItem { + uri: string; + value: string; + // Additional URIs that contribute the same code snippet. + additionalUris?: string[]; +} + +// Relevant diagnostic from a given resource. The URI is used for content exclusion. +export interface DiagnosticBag extends ContextItem { + uri: Uri; + values: Diagnostic[]; +} + +export type SupportedContextItem = Trait | CodeSnippet | DiagnosticBag; +export type SupportedContextItemType = 'Trait' | 'CodeSnippet' | 'DiagnosticBag'; +export type ContextItemOrigin = 'request' | 'update'; +export namespace ContextItemOrigin { + export function is(value: string): value is ContextItemOrigin { + return value === 'request' || value === 'update'; + } +} diff --git a/completions-sample-code/vscode-node/types/src/core.ts b/completions-sample-code/vscode-node/types/src/core.ts new file mode 100644 index 0000000..b09de91 --- /dev/null +++ b/completions-sample-code/vscode-node/types/src/core.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Type } from '@sinclair/typebox'; + +export { + CancellationToken, + CancellationTokenSource, + Command, + Disposable, + DocumentUri, + + Position, + + Range, + + TextDocumentItem, + TextEdit, + VersionedTextDocumentIdentifier, + + WorkspaceFolder +} from 'vscode-languageserver-protocol'; + +const PositionSchema = Type.Object({ + line: Type.Integer({ minimum: 0 }), + character: Type.Integer({ minimum: 0 }), +}); + +export const RangeSchema = Type.Object({ + start: PositionSchema, + end: PositionSchema, +}); \ No newline at end of file diff --git a/completions-sample-code/vscode-node/types/src/index.ts b/completions-sample-code/vscode-node/types/src/index.ts new file mode 100644 index 0000000..1131691 --- /dev/null +++ b/completions-sample-code/vscode-node/types/src/index.ts @@ -0,0 +1,12 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export { CancellationToken, CancellationTokenSource, Disposable, Position, Range, TextEdit } from 'vscode-languageserver-protocol'; +export * from './auth'; +export * from './codeCitation'; +export * from './contextProviderApiV1'; +export * from './core'; +export * from './status'; + diff --git a/completions-sample-code/vscode-node/types/src/status.ts b/completions-sample-code/vscode-node/types/src/status.ts new file mode 100644 index 0000000..17ae4d0 --- /dev/null +++ b/completions-sample-code/vscode-node/types/src/status.ts @@ -0,0 +1,20 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + + +/** + * Status of the agent, used in different IDE status menus and icons. + * + * **Normal** - When everything is working normally (*Current Default*). + * + * **InProgress** - When a task is in progress. When a spinner should be shown. + * + * **Error** - When cannot connect, is not authorized, or authenticated. + * + * **Warning** - When there is a temporary issue. Such as request failed or logged out unexpectedly. + * + * **Inactive** - When the current file is ignored due to file size or content exclusions. + */ +export type StatusKind = 'Normal' | 'Error' | 'Warning' | 'Inactive'; diff --git a/milkdown-docs/blogs/announcing-telemetry-inspector.md b/milkdown-docs/blogs/announcing-telemetry-inspector.md new file mode 100644 index 0000000..5dc9d14 --- /dev/null +++ b/milkdown-docs/blogs/announcing-telemetry-inspector.md @@ -0,0 +1,142 @@ +# Announcing Telemetry Inspector + +There's a lot of questions from community asking that how can they know what plugins are enabled. +From Milkdown@7.2, we've added telemetries for milkdown, it can be available by inspectors. + +With this API, you can inspect editor inner status. +You can even use visualizer to visualize the data. We create a simple example on [our playground](/playground). + +![Milkdown Inspector](/blogs/announcing-telemetry-inspector/milkdown-inspector.gif) + +## Get Started + +Inspector will be a top-level API in Milkdown. You can use it like this: + +```ts +import { Editor } from "@milkdown/core"; +import { Telemetry } from "@milkdown/ctx"; + +const editor = await Editor.make() + // Inspector is disabled by default considering performance. You need to enable it manually. + .enableInspector() + // ... + .create(); + +const telemetry: Telemetry[] = editor.inspect(); +``` + +The `Telemetry` interface will have the following fields: + +```ts +interface Telemetry { + // User defined information for the plugin. + metadata: Meta; + + // The slices and their current value defined by the plugin. + injectedSlices: { name: string; value: unknown }[]; + + // The slices and their current value consumed by the plugin. + consumedSlices: { name: string; value: unknown }[]; + + // The timers and their duration defined by the plugin. + recordedTimers: { name: string; duration: number; status: TimerStatus }[]; + + // The timers and their duration consumed by the plugin. + // Generally, the plugin will wait for them. + waitTimers: { name: string; duration: number; status: TimerStatus }[]; +} + +type TimerStatus = "pending" | "resolved" | "rejected"; + +interface Meta { + displayName: string; + description?: string; + package: string; + group?: string; + additional?: Record<string, any>; +} +``` + +For every plugin, it'll have a telemetry if it has metadata declared. +With the data, you'll know the sequence of the plugins loaded, the slices and timers they defined and consumed. +For example: + +```ts +[ + { + metadata: { + displayName: "Config", + package: "@milkdown/core", + group: "System", + }, + injectedSlices: [], + consumedSlices: [ + /* ... */ + ], + recordedTimers: [ + { + name: "ConfigReady", + duration: 3, + status: "resolved", + }, + ], + waitTimers: [], + }, + { + metadata: { + displayName: "Init", + package: "@milkdown/core", + group: "System", + }, + injectedSlices: [], + consumedSlices: [ + /* ... */ + ], + recordedTimers: [ + { + name: "InitReady", + duration: 5, + status: "resolved", + }, + ], + waitTimers: [ + { + name: "ConfigReady", + duration: 5, + status: "resolved", + }, + ], + }, +]; +``` + +From above information, we can know that the `Init` plugin wait for `Config` plugin to be ready. +We can build a sequence diagram from the data. + +![Timer Sequence](/blogs/announcing-telemetry-inspector/timer-sequence.gif) + +## Add Metadata for Plugin + +For plugin maintainers, you can add metadata to your plugin to make it more friendly to the inspector. + +```ts +import { MilkdownPlugin } from "@milkdown/ctx"; + +const yourMilkdownPlugin: MilkdownPlugin = () => { + /* your implementation */ +}; + +yourMilkdownPlugin.metadata = { + displayName: "Your Plugin", + package: "your-plugin-package", + description: "Your plugin description", + group: "If you have a lot of plugins in your package, you can group them.", + addtitional: { + /* You can add any additional information here. */ + version: "1.0.0", + authror: "Mike", + }, +}; +``` + +With metadata, your plugin will report telemetry correctly to the inspector. diff --git a/milkdown-docs/blogs/build-your-own-milkdown-copilot.md b/milkdown-docs/blogs/build-your-own-milkdown-copilot.md new file mode 100644 index 0000000..51b50cd --- /dev/null +++ b/milkdown-docs/blogs/build-your-own-milkdown-copilot.md @@ -0,0 +1,247 @@ +# Build Your Own Milkdown Copilot + +OpenAI introduced ChatGPT in 2020, which is a chatbot that can generate natural language responses to user input. +Which brings us a new way to interact with devices and applications. +Nowadays, there are more and more tools that are powered by AI. Such as Notion, GitHub and even Microsoft 365. + +Since OpenAI also released the [API](https://openai.com/blog/openai-api) of it. And Milkdown is composed by plugins. +I think it's possible to build a Milkdown Copilot Plugin that can help you write documents. So I did it. +Let's see the result. + +![Milkdown Copilot](/blogs/build-your-own-milkdown-copilot/milkdown-copilot.gif) + +Looks cool, right? But how does it work? I'll explain it in the following sections. + +## Prepare a Backend + +**Before we start, you need to have a OpenAI API Key.** You'll need to get one [here](https://platform.openai.com/account/api-keys). +I'll not explain how to get it. You can find the details in their [official docs](https://platform.openai.com/). + +I'll use Node.js to build the backend. You can use any language you like. +The backend is very simple. It just calls the OpenAI API and returns the result. + +```ts +import { Configuration, OpenAIApi } from "openai"; + +const configuration = new Configuration({ + // Get your API key from env variable + apiKey: process.env.OPENAPI_KEY, +}); +const openai = new OpenAIApi(configuration); + +export const handler = async (req, res, next) => { + if (req.path === "/api/copilot" && req.method === "POST") { + const buffers = []; + + // Get the body of the request. + const body = JSON.parse(req.body); + + // Get prompt from the body. + const { prompt } = body; + const completion = await openai.createCompletion({ + // Pick a model you like + model: "text-davinci-003", + prompt, + }); + const hint = completion.data.choices[0].text; + return res.end(JSON.stringify({ hint })); + } + next(); + return; +}; +``` + +We watch the `/api/copilot` route and call the OpenAI API when we receive a POST request. +The post request should contain a `prompt` field which is the text that we want to complete. + +To call our API, we just need one single helper in browser environment: + +```ts +async function fetchAIHint(prompt: string) { + const data: Record<string, string> = { prompt }; + const response = await fetch("/api/copilot", { + method: "POST", + body: JSON.stringify(data), + }); + const res = (await response.json()) as { hint: string }; + return res.hint; +} +``` + +## Build a Milkdown Plugin + +Now let's focus on the Milkdown Copilot Plugin. + +Basically I want to implement two things: + +1. When the user types `<Enter>` or `<Space>`, they will get a hint from the copilot. +2. When the user types `<Tab>`, they will apply the content from the hint to the editor. + +### Overview + +To build a bridge between the copilot and the editor, +we can build a prosemirror plugin and use the `onKeyDown` hook to listen to the keydown event. + +```ts +function keyDownHandler(ctx: Ctx, event: Event) { + if (event.key === "Enter" || event.code === "Space") { + getHint(ctx); + return; + } + if (event.key === "Tab") { + // prevent the browser from focusing on the next element. + event.preventDefault(); + + applyHint(ctx); + return; + } + + hideHint(ctx); +} +``` + +When the user types `<Enter>` or `<Space>`, we will call the `getHint` function to get a hint from the copilot. +And when the user types `<Tab>`, we will call the `applyHint` function to apply the hint to the editor. +If user types other keys, we will hide the hint. + +And we also need a component to render the hint. Here I choose to use a simple [widget decoration in prosemirror](https://prosemirror.net/docs/ref/#view.Decoration^widget). + +```ts +function renderHint(message: string) { + const dom = document.createElement("pre"); + dom.className = "copilot-hint"; + dom.innerHTML = message; + return dom; +} +``` + +So our component looks like: + +```ts +import { Plugin, PluginKey } from "@milkdown/prose/state"; +import { Decoration, DecorationSet } from "@milkdown/prose/view"; +import { $prose } from "@milkdown/utils"; + +const initialState = { + deco: DecorationSet.empty, + message: "", +}; + +export const copilotPluginKey = new PluginKey("milkdown-copilot"); +export const copilotPlugin = $prose( + (ctx) => + new Plugin({ + key: copilotPluginKey, + props: { + handleKeyDwon(view, event) { + keydownHandler(ctx, event); + }, + decorations(state) { + return copilotPluginKey.getState(state).deco; + }, + }, + state: { + init() { + return { ...initialState }; + }, + apply(tr, value, _prevState, state) { + const message = tr.getMeta(copilotPluginKey); + if (typeof message !== "string") return value; + + if (message.length === 0) { + return { ...initialState }; + } + + const { to } = tr.selection; + const widget = Decoration.widget(to + 1, () => renderHint(message)); + return { + deco: DecorationSet.create(state.doc, [widget]), + message, + }; + }, + }, + }), +); +``` + +### Get Hint + +To get a hint from the copilot, we need to get the text before the cursor. + +```ts +function getHint(ctx: Ctx) { + const view = ctx.get(editorViewCtx); + const { state } = view; + const { tr, schema } = state; + const { from } = tr.selection; + + const slice = tr.doc.slice(0, from); + const serializer = ctx.get(serializerCtx); + const doc = schema.topNodeType.createAndFill(undefined, slice.content); + if (!doc) return; + + const markdown = serializer(doc); + fetchAIHint(markdown).then((hint) => { + const tr = view.state.tr; + view.dispatch(tr.setMeta(copilotPluginKey, hint)); + }); +} +``` + +1. First of all, we get the `selection` from the `state` of the editor. +2. Then we get a `slice` of the document from the start to the cursor. +3. Then we use the `serializer` to convert the slice to markdown. +4. After that, we call the `fetchAIHint` function to get a hint from the copilot. +5. Finally, we dispatch a transaction with the hint message we get to update the state of the editor. + +### Hide Hint + +To hide the hint, we just need to dispatch a transaction with an empty message. + +```ts +function hideHint(ctx: Ctx) { + const view = ctx.get(editorViewCtx); + const { state } = view; + const { tr } = state; + view.dispatch(tr.setMeta(copilotPluginKey, "")); +} +``` + +### Apply Hint + +Since we pass markdown to the OpenAI API. It may return a markdown snippet. + +So, before we apply the hint to the editor, we need to convert the markdown snippet to prosemirror node. + +```ts +function applyHint(ctx: Ctx) { + const view = ctx.get(editorViewCtx); + const { state } = view; + const { tr, schema } = state; + + const { message } = copilotPluginKey.getState(state); + const parser = ctx.get(parserCtx); + const slice = parser(message); + const dom = DOMSerializer.fromSchema(schema).serializeFragment(slice.content); + const node = DOMParser.fromSchema(schema).parseSlice(dom); + + // Reset the hint since it's applied + tr.setMeta(copilotPluginKey, "") + // Replace the selection with the hint + .replaceSelection(node); + + view.dispatch(tr); +} +``` + +1. First of all, we get the hint message from the state of the editor. +2. Then we use the `parser` to convert the markdown snippet to prosemirror node. +3. Finally, we dispatch a transaction to replace the selection with the hint. + +## Conclusion + +In this article, we have built a really simple Copilot plugin for Milkdown. +The plugin is not perfect, but it's a good start to help you build your own. + +The source code is available on [Milkdown/examples/vanilla-openapi](https://github.com/Milkdown/examples/tree/main/vanilla-openai). +I hope it can give you some inspiration. diff --git a/milkdown-docs/blogs/introducing-milkdown@7.md b/milkdown-docs/blogs/introducing-milkdown@7.md new file mode 100644 index 0000000..64c4a81 --- /dev/null +++ b/milkdown-docs/blogs/introducing-milkdown@7.md @@ -0,0 +1,151 @@ +# Introducing Milkdown@7 + +It's been almost one year since the release of [milkdown](https://milkdown.dev) V6. +It helped a lot of users to build their own markdown based applications. +It has 13k downloads per month and I feel so grateful that users like that. + +However, we noticed that there're some problems cannot be resolved if we don't make a new major version. +What big changes did we made? I'll introduce them to you in this blog. + +## TL;DR + +- The editor becomes a first-class headless component. + +- Factory plugins are fully replaced by **composable plugins**. + +- Runtime plugin toggling is supported. + +- Universal widget plugins. + +- Better Vue and React support. + +- API documentation is provided. + +## Why Headless? + +In the past, milkdown had a lot of internal styles to make sure the editor can work out of box and the themes are easy to create. +However, I found it limits the users to design their own editor. +Even worse, if you have an well designed application, +it is really hard to keep the style of the milkdown editor same with the rest of the application. +You'll need to override lots of styles everywhere. +It stops a log of users from using milkdown. + +If we think about why users need an editor, +the most important thing is always the functionality of the editor. +Users just want a component that can provide smooth editing experience. +Style will always be the second thing. + +So, why not remove all the internal styles and make the editor a headless component? +The users can easily integrate the editor into their own application. +They can use their own styles and even use their own components to render the editor. +We just care about the functionality of the editor. Make sure it works well. + +## Composable Plugins + +Although the composable plugins have been existed in milkdown for a long time, +we use factory plugins to create most of the official plugins in V6. + +But, the problem is that factory plugins limit the possibility of the plugins. +The factory plugins handle a bunch of complex logic and it is hard to extend. +So for users who want to create a plugin in a easy way, they must follow the factory plugin's way. + +```ts +const nodePlugin = createPlugin(() => ({ + id: 'node', + schema: someSchema, + inputRules: someInputRules + commands: someCommands +})) +``` + +See? You can define a lot of things inside the factory plugin. +But if you want to use some part of them in another plugin, it's really hard to do that. + +However, the milkdown's plugin system is designed to be flexible and composable. +We want to let users to control the data flow entirely. + +So, we decided to remove all the factory plugins and use composable plugins to replace them. +The composable plugins can keep the atomicity of the plugins and make the plugin system more flexible. +They also make the plugin system easier to maintain. + +```ts +const nodeSchema = $node("node", someSchema); +const nodeInputRules = $inputRules(someInputRules); +const nodeCommands = $commands(someCommands); +``` + +If you want to reuse them, it also will be very easy. + +```ts +const anotherCommand = $commands(() => { + return setBlockType(nodeSchema.type()); +}); +``` + +## Runtime Plugin Toggling + +In the past, once you register a plugin, you cannot remove it. +In V7, we support runtime plugin toggling by providing two new API: `editor.remove` and `editor.removeConfig`. +They can let users remove the plugins and configs at runtime. + +```ts +import { Editor } from "@milkdown/core"; +import { someMilkdownPlugin } from "some-milkdown-plugin"; + +const editor = await Editor.config(configForPlugin) + .use(someMilkdownPlugin) + .create(); + +// remove plugin +await editor.remove(someMilkdownPlugin); + +// remove config +editor.removeConfig(configForPlugin); + +// add another plugin +editor.use(anotherMilkdownPlugin); + +// Recreate the editor to apply changes. +await editor.create(); +``` + +Also, if you call the `editor.create` method after the editor is created, +it will recreate the editor and apply all the changes. + +## Universal Widget Plugins + +We have 4 official widget plugins in V6: _slash_, _tooltip_, _block_ and _menu_. + +They are all well designed and easy to use. +But if you want to customize them, what you can do is really limited. +Also, it's hard to reuse their logic even if you want to create something similar to them. +For example, if you want to create a mention plugin which will show a list of users when you type `@`, +you need to create a new plugin from scratch. + +So, in V7, we make _slash_, _tooltip_ and _block_ plugins universal. +You can use them to build you features easily. +For example, if you want to create a mention plugin, you can use the new slash plugin to do that. +Another example is that you can also create tooltips for different types of nodes. +Display a tooltip with input when you focus on an image node, or display a tooltip with buttons when you select some text. + +What about the _menu_ plugin? We removed it because we think it's easy to create a menu plugin by yourself. +We've already done that in the [official playground](https://milkdown.dev/playground). +And, trust me, [it won't need much code](https://github.com/milkdown/website/blob/main/src/component/Playground/Milkdown/index.tsx#L57). + +## Better Vue and React Support + +Thanks to the [Saul-Mirone/prosemirror-adapter project](https://github.com/Saul-Mirone/prosemirror-adapter). +In milkdown V7. We allow users to use vue and react to render lots of parts of the editor. +For example, you can use them to render your own code block, drag handle or even small icons. + +- React Example: [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/Milkdown/examples/tree/main/react-custom-component) +- Vue Example: [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/Milkdown/examples/tree/main/vue-custom-component) + +## API Documentation + +What's the hardest thing to do when maintaining an open source project? +Keep the documentation up to date. +Thanks to the [marijnh/builddocs project](https://github.com/marijnh/builddocs), +we can generate the API documentation automatically from the source code. + +We also redesigned the documentation website, provide a more powerful playground and lots of examples. diff --git a/milkdown-docs/blogs/understanding-headless-slash-plugin.md b/milkdown-docs/blogs/understanding-headless-slash-plugin.md new file mode 100644 index 0000000..5308cdd --- /dev/null +++ b/milkdown-docs/blogs/understanding-headless-slash-plugin.md @@ -0,0 +1,172 @@ +# Understanding Headless Slash Plugin + +In the old Milkdown versions. The slash plugin can be used to display a list of commands when users type `/` in the editor. +It provides a way to insert nodes and commands into the editor, and it's really easy to use. + +![legacy slash plugin](/blogs/understanding-headless-slash-plugin/legacy-slash-plugin.png) + +However, it's hard to extend the slash plugin to support more commands, or if you want to change the UI of the slash plugin, you have to rewrite the whole plugin. +But, write a new plugin is always a hard work. You have to understand a lot of context and APIs of both ProseMirror and Milkdown. + +## User Story + +So, why don't we provide the slash plugin as a headless plugin? +In most cases, developers just want to make sure that when users type a special character, a dropdown menu will be displayed. +But the trigger character and the UI of the dropdown menu are different in different cases. + +For example: + +- When user type `/`, the menu contains a list of **commands**. +- When user type `:`, the menu contains a list of **emoji**. +- When user type `@`, the menu contains a list of **users**. + +That's the story behind the headless slash plugin. We provide the plugin to solve a single problem: **display a dropdown menu when users input satisfy a condition**. + +## How to use + +In the new slash plugin, you'll need to control when to display the dropdown menu by yourself. +And you'll also need to provide the UI of the dropdown menu. +So, you'll need to create a `SlashProvider` instance. + +```ts +import { slashPlugin, SlashProvider } from "@milkdown/plugin-slash"; + +const slashProvider = new SlashProvider({ + content: YourDropdownUI, + shouldShow(this: SlashProvider, view: EditorView) { + const currentText = this.getContent(view); + + if (currentText === "") { + return false; + } + + // Display the menu if the last character is `/`. + if (currentText.endsWith("/")) { + return true; + } + + return false; + }, +}); +``` + +Then, you can use the slash provider in your plugin view. + +```ts +import { EditorState } from "@milkdown/prose/state"; +import { EditorView, PluginView } from "@milkdown/prose/view"; + +function yourSlashView(): PluginView { + return { + update: (view: EditorView, prevState: EditorState) => { + slashProvider.update(view, prevState); + }, + destroy: () => { + slashProvider.destroy(); + }, + }; +} +``` + +Last, you'll need to add the slash plugin to your editor. + +```ts +import { Editor } from "@milkdown/core"; +import { slashFactory } from "@milkdown/plugin-slash"; + +const slash = slashFactory("my-slash"); + +Editor.make() + .config((ctx) => { + ctx.set(slash.key, { + view: slashPluginView, + }); + }) + .use(slash) + .create(); +``` + +## Use with Prosemirror Adapter + +If you're using milkdown with UI frameworks like React, +I recommend you to use the [Prosemirror Adapter](https://github.com/Saul-Mirone/prosemirror-adapter). +It can help you build prosemirror UI components with your favorite UI framework. + +For example, if you're using React: + +```tsx +import { SlashProvider } from "@milkdown/plugin-slash"; +import { useInstance } from "@milkdown/react"; +import { usePluginViewContext } from "@prosemirror-adapter/react"; + +export const DropdownMenu = () => { + const { view, prevState } = usePluginViewContext(); + const slashProvider = useRef<SlashProvider>(); + const divRef = useRef<HTMLDivElement>(null); + const [loading] = useInstance(); + + useEffect(() => { + if (!ref.current || loading) return; + + slashProvider.current ??= new SlashProvider({ + content: divRef.current, + // ... + }); + + return () => { + slashProvider.current?.destroy(); + slashProvider.current = undefined; + }; + }, [loading, root, setOpened, setSearch, setSelected]); + + useEffect(() => { + slashProvider.current?.update(view, prevState); + }); + + // Add a wrapper `div` to hide the dropdown menu when initializing. + return ( + <div className="hidden"> + <div role="tooltip" ref={divRef}> + <h1>Hi! I'm a dropdown menu.</h1> + </div> + </div> + ); +}; +``` + +And in your editor component: + +```ts +import { usePluginViewFactory } from "@prosemirror-adapter/react"; + +export const YourEditor = () => { + const pluginViewFactory = usePluginViewFactory(); + + useEditor((editor) => { + return Editor.make() + .config((ctx) => { + ctx.set(slash.key, { + view: pluginViewFactory({ + component: DopdownMenu, + }), + }); + }) + .use(slash); + }); + + // ... +}; +``` + +## Real World Example + +In [milkdown playground](/playground), you can type `/` to display a dropdown menu. + +![command dropdown](/blogs/understanding-headless-slash-plugin/command-dropdown.png) + +You can also type `:(\S)+` (for example: `:mil`) to display a list of emojis. + +![emoji dropdown](/blogs/understanding-headless-slash-plugin/emoji-dropdown.png) + +You can find the source code of them in [Milkdown website](https://github.com/Milkdown/website). +I hope you enjoy the new slash plugin. diff --git a/milkdown-docs/guide/architecture-overview.md b/milkdown-docs/guide/architecture-overview.md new file mode 100644 index 0000000..bf205e8 --- /dev/null +++ b/milkdown-docs/guide/architecture-overview.md @@ -0,0 +1,199 @@ +# Architecture Overview + +Milkdown is built with a modular, layered architecture that provides flexibility and extensibility. This document explains the core architectural concepts and how they work together. + +![0.75](/guide/milkdown-architecture.png "Milkdown Architecture") + +## Core Architecture Layers + +Milkdown's architecture is built upon four distinct layers, each providing specific functionality and extensibility: + +### ๐Ÿฅ› Core Layer + +The foundation of Milkdown that provides: + +- Plugin loading and management system +- Core editor concepts and interfaces +- Base document model integration +- Essential utilities and helpers + +### ๐Ÿง‡ Plugin Layer + +A comprehensive collection of modular plugins that extend the editor's functionality: + +- Syntax plugins (Markdown parsing, GFM, etc.) +- UI plugins (toolbar, menu, etc.) +- Feature plugins (image upload, table, etc.) +- Utility plugins (history, clipboard, etc.) + +### ๐Ÿฎ Component Layer + +Headless UI components that serve as building blocks: + +- Toolbar components +- Slash menu components +- Table components + +### ๐Ÿฐ Editor Layer + +Ready-to-use, user-friendly editors: + +- Crepe editor +- Custom editor implementations + +## Architecture Benefits + +This layered approach provides several key benefits: + +1. **Modularity**: Each layer can be used independently +2. **Flexibility**: Mix and match components as needed +3. **Extensibility**: Create custom implementations at any layer +4. **Maintainability**: Clear separation of concerns +5. **Reusability**: Components can be shared across implementations + +## Markdown Transformation + +![0.75](/guide/transformer.png "Transformer") + +Milkdown's transformation system handles the conversion between Markdown and the editor's internal document model: + +### Parsing Process + +1. Markdown text โ†’ Remark AST +2. Remark AST โ†’ ProseMirror Schema +3. Schema โ†’ ProseMirror Document + +### Serialization Process + +1. ProseMirror Document โ†’ ProseMirror Schema +2. Schema โ†’ Remark AST +3. Remark AST โ†’ Markdown text + +This transformation system ensures: + +- Accurate Markdown parsing +- Consistent document structure +- Reliable serialization +- Extensible transformation pipeline + +## Context System + +The Context System is a powerful state management and dependency coordination system that enables plugins to work together seamlessly. + +![1.00](/guide/plugin-sequence.png "Plugin Sequence") + +### Core Concepts + +#### 1. Context (Ctx) + +The main interface for plugins to interact with the system: + +```typescript +interface Ctx { + get: <T>(slice: Slice<T>) => T; + set: <T>(slice: Slice<T>, value: T) => void; + wait: (timer: Timer) => Promise<void>; + done: (timer: Timer) => void; + inject: <T>(slice: Slice<T>, value: T) => void; + remove: <T>(slice: Slice<T>) => void; +} +``` + +#### 2. Slices + +State containers that can be shared between plugins: + +```typescript +// Create a slice with initial value and name +const themeSlice = createSlice("light", "theme"); + +// Use in a plugin +const themePlugin: MilkdownPlugin = (ctx) => { + return () => { + // Read current theme + const theme = ctx.get(themeSlice); + + // Update theme + ctx.set(themeSlice, "dark"); + + // React to theme changes + ctx.watch(themeSlice, (newTheme) => { + // Handle theme change + }); + }; +}; +``` + +#### 3. Timers + +Dependency management system for plugin coordination: + +```typescript +// Define a timer +const dataReady = createTimer("DataReady"); + +// Use in a plugin +const dataPlugin: MilkdownPlugin = (ctx) => { + ctx.record(dataReady); + + return async () => { + // Wait for dependencies + await ctx.wait(SchemaReady); + + // Do work + // ... + + // Mark as ready + ctx.done(dataReady); + }; +}; +``` + +### Plugin Lifecycle + +Plugins follow a consistent lifecycle pattern: + +```typescript +const examplePlugin: MilkdownPlugin = (ctx) => { + // 1. Setup Phase + ctx.inject(mySlice, defaultValue); + ctx.record(myTimer); + + return async () => { + // 2. Initialization Phase + await ctx.wait(RequiredTimer); + + // 3. Runtime Phase + const value = ctx.get(mySlice); + ctx.set(mySlice, newValue); + + // 4. Cleanup Phase + return () => { + ctx.remove(mySlice); + }; + }; +}; +``` + +### Best Practices + +1. **State Management** + - Use slices for shared state + - Keep state minimal and focused + - Watch for state changes when needed + +2. **Dependency Management** + - Use timers for coordination + - Wait for required dependencies + - Mark completion appropriately + +3. **Plugin Organization** + - Follow the lifecycle pattern + - Clean up resources properly + - Document dependencies clearly + +## Next Steps + +- Start to [use Crepe editor](/docs/guide/using-crepe) +- Learn more about [writing plugins](/docs/plugin/plugins-101) +- Explore [available plugins](/docs/plugin/using-plugins) diff --git a/milkdown-docs/guide/code-highlighting.md b/milkdown-docs/guide/code-highlighting.md new file mode 100644 index 0000000..ef41668 --- /dev/null +++ b/milkdown-docs/guide/code-highlighting.md @@ -0,0 +1,160 @@ +# Code Highlighting + +Milkdown supports syntax highlighting for code blocks through the `@milkdown/plugin-highlight` plugin. This plugin provides several options for highlighting code with different syntax highlighters. + +## Installation + +```bash +npm install @milkdown/plugin-highlight +``` + +## Basic Usage + +The highlight plugin requires a parser to be configured. Here's a basic example using the Shiki parser: + +```typescript +import { Editor } from "@milkdown/core"; +import { commonmark } from "@milkdown/preset-commonmark"; +import { highlight, highlightPluginConfig } from "@milkdown/plugin-highlight"; +import { createParser } from "@milkdown/plugin-highlight/shiki"; + +const editor = Editor.make() + .config(async (ctx) => { + const parser = await createParser({ + theme: "github-light", + langs: ["javascript", "typescript", "python", "html", "css"], + }); + ctx.set(highlightPluginConfig.key, { parser }); + }) + .use(commonmark) + .use(highlight) + .create(); +``` + +## Available Parsers + +The plugin supports multiple syntax highlighting libraries: + +### Shiki + +Provides high-quality syntax highlighting with VS Code themes. Learn more at [Shiki](https://shiki.style/): + +```typescript +import { createParser } from "@milkdown/plugin-highlight/shiki"; + +const parser = await createParser({ + theme: "github-light", + langs: ["javascript", "typescript", "python"], +}); +ctx.set(highlightPluginConfig.key, { parser }); +``` + +### Lowlight + +Based on [highlight.js](https://highlightjs.org/), supports many languages: + +```typescript +import { createParser } from "@milkdown/plugin-highlight/lowlight"; +import { common } from "lowlight"; + +const parser = createParser({ common }); +ctx.set(highlightPluginConfig.key, { parser }); +``` + +Learn more about Lowlight at [lowlight](https://github.com/wooorm/lowlight). + +### Refractor + +Based on [Prism.js](https://prismjs.com/): + +```typescript +import { createParser } from "@milkdown/plugin-highlight/refractor"; +import { refractor } from "refractor"; + +const parser = createParser({ refractor }); +ctx.set(highlightPluginConfig.key, { parser }); +``` + +Learn more about Refractor at [refractor](https://github.com/wooorm/refractor). + +### Sugar High + +A lightweight and fast syntax highlighter. Learn more at [Sugar High](https://github.com/huozhi/sugar-high): + +```typescript +import { createParser } from "@milkdown/plugin-highlight/sugar-high"; + +const parser = createParser(); +ctx.set(highlightPluginConfig.key, { parser }); +``` + +## Styling + +The highlighted code will have CSS classes applied based on the chosen parser. You'll need to include appropriate CSS to style the highlighted tokens. + +### Sugar High Classes + +Sugar High uses classes like: + +- `sh__token--identifier` +- `sh__token--string` +- `sh__token--keyword` +- `sh__token--sign` +- `sh__token--property` + +You can style these using CSS variables: + +```css +.sh__token--identifier { + color: var(--sh-identifier); +} +.sh__token--string { + color: var(--sh-string); +} +.sh__token--keyword { + color: var(--sh-keyword); +} +``` + +### Other Parsers + +For Lowlight, Refractor, and Shiki, refer to their respective documentation for styling information. + +## Example + +Here's a complete example with Shiki: + +```typescript +import { Editor } from "@milkdown/core"; +import { commonmark } from "@milkdown/preset-commonmark"; +import { highlight, highlightPluginConfig } from "@milkdown/plugin-highlight"; +import { createParser } from "@milkdown/plugin-highlight/shiki"; + +async function createHighlightedEditor() { + const parser = await createParser({ + theme: "github-light", + langs: ["javascript", "typescript", "python", "html", "css", "json"], + }); + + const editor = Editor.make() + .config((ctx) => { + ctx.set(highlightPluginConfig.key, { parser }); + }) + .use(commonmark) + .use(highlight); + + await editor.create(); + return editor; +} +``` + +With this setup, your code blocks will be automatically highlighted: + +````markdown +```javascript +console.log("Hello, world!"); +const greeting = (name) => `Hello, ${name}!`; +``` +```` + +The code above will render with syntax highlighting applied to keywords, strings, and other language constructs. diff --git a/milkdown-docs/guide/collaborative-editing.md b/milkdown-docs/guide/collaborative-editing.md new file mode 100644 index 0000000..be552dc --- /dev/null +++ b/milkdown-docs/guide/collaborative-editing.md @@ -0,0 +1,122 @@ +# Collaborative Editing + +Milkdown supports collaborative editing powered by [Y.js](https://docs.yjs.dev/). +We provide the [@milkdown/plugin-collab](/docs/api/plugin-collab) plugin to help you use milkdown with yjs easily. +This plugin includes basic collaborative editing features like: + +- Sync between clients. +- Remote cursor support. +- Undo/Redo support. + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vanilla-collab"} + +## Configure Plugin + +First you need to install the plugin and yjs through npm: + +```bash +npm install @milkdown/plugin-collab + +npm install yjs y-protocols y-prosemirror +``` + +And you also need to choose a [provider for yjs](https://docs.yjs.dev/ecosystem/connection-provider), here we use [y-websocket](https://docs.yjs.dev/ecosystem/connection-provider/y-websocket) as an example. + +After the installation, you can configure your editor: + +```typescript +// ...import other plugins +import { collab, collabServiceCtx } from "@milkdown/plugin-collab"; + +async function setup() { + const editor = await Editor.make() + .config(nord) + .use(commonmark) + .use(collab) + .create(); + + const doc = new Doc(); + const wsProvider = new WebsocketProvider("<YOUR_WS_HOST>", "milkdown", doc); + + editor.action((ctx) => { + const collabService = ctx.get(collabServiceCtx); + + collabService + // bind doc and awareness + .bindDoc(doc) + .setAwareness(wsProvider.awareness) + // connect yjs with milkdown + .connect(); + }); +} +``` + +Now your editor can support collaborative editing. Isn't it easy? + +## Connect and Disconnect + +You may want to control the connect status of the editor manually. + +```typescript +editor.action((ctx) => { + const collabService = ctx.get(collabServiceCtx); + const doc = new Doc(); + const wsProvider = new WebsocketProvider("<YOUR_WS_HOST>", "milkdown", doc); + + collabService.bindDoc(doc).setAwareness(wsProvider.awareness); + + document.getElementById("connect").onclick = () => { + wsProvider.connect(); + collabService.connect(); + }; + + document.getElementById("disconnect").onclick = () => { + wsProvider.disconnect(); + collabService.disconnect(); + }; +}); +``` + +## Default Template + +By default, the editor will show a empty document. You may want to use a template to show a document. + +```typescript +const template = `# Heading`; + +editor.action((ctx) => { + const collabService = ctx.get(collabServiceCtx); + const doc = new Doc(); + const wsProvider = new WebsocketProvider("<YOUR_WS_HOST>", "milkdown", doc); + + collabService.bindDoc(doc).setAwareness(wsProvider.awareness); + + wsProvider.once("synced", async (isSynced: boolean) => { + if (isSynced) { + collabService + // apply your template + .applyTemplate(markdown) + // don't forget connect + .connect(); + } + }); +}); +``` + +Keep in mind that applying a template multiple times may cause some unexpected behavior, such as duplicate content. +Because of this you need to make sure **the template is applied only once**. + +By default, the template will only be applied if _document get from remote server is empty_. +You can control this behavior through passing second parameter to `applyTemplate`: + +```typescript +collabService + .applyTemplate(markdown, (remoteNode, templateNode) => { + // return true to apply template + }) + // don't forget connect + .connect(); +``` + +Here the nodes we get are [prosemirror nodes](https://prosemirror.net/docs/ref/#model.Node). +You should return `true` if the template should be applied, and `false` if not. diff --git a/milkdown-docs/guide/commands.md b/milkdown-docs/guide/commands.md new file mode 100644 index 0000000..0960db2 --- /dev/null +++ b/milkdown-docs/guide/commands.md @@ -0,0 +1,250 @@ +# Commands + +Commands are a powerful way to programmatically modify editor content. The command system in Milkdown provides a flexible and type-safe way to create, manage, and execute commands. + +## Command Manager + +--- + +The command manager is the central place for handling all editor commands. It provides methods to: + +- Register new commands +- Execute commands +- Chain multiple commands together +- Handle command arguments + +## Run a Command + +--- + +You can execute commands using the command manager through the editor's action system: + +```typescript +import { Editor, commandsCtx } from "@milkdown/kit/core"; +import { + commonmark, + toggleEmphasisCommand, +} from "@milkdown/kit/preset/commonmark"; + +async function setup() { + const editor = await Editor.make().use(commonmark).create(); + + const toggleItalic = () => + editor.action((ctx) => { + // get command manager + const commandManager = ctx.get(commandsCtx); + + // call command + commandManager.call(toggleEmphasisCommand.key); + }); + + // get markdown string: + $button.onClick = toggleItalic; +} +``` + +## Command Chaining + +--- + +You can chain multiple commands together using the command manager's `chain` method. Commands in the chain will be executed in order until one of them returns `true`: + +```typescript +import { Editor, commandsCtx } from "@milkdown/kit/core"; +import { + commonmark, + toggleEmphasisCommand, + toggleStrongCommand, +} from "@milkdown/kit/preset/commonmark"; + +const editor = await Editor.make().use(commonmark).create(); + +editor.action((ctx) => { + const commandManager = ctx.get(commandsCtx); + + // Chain multiple commands + commandManager + .chain() + .pipe(toggleEmphasisCommand.key) // Try to toggle emphasis + .pipe(toggleStrongCommand.key) // If emphasis fails, try to toggle strong + .run(); +}); +``` + +You can also mix inline commands with registered commands: + +```typescript +import { chainCommands } from "@milkdown/prose/commands"; + +editor.action((ctx) => { + const commandManager = ctx.get(commandsCtx); + + commandManager + .chain() + .inline(someInlineCommand) // Add an inline command + .pipe(toggleEmphasisCommand.key) // Add a registered command + .run(); +}); +``` + +## Create a Command + +--- + +To create a command, use the `$command` utility from `@milkdown/utils`. Commands should be [prosemirror commands](https://prosemirror.net/docs/guide/#commands). + +### Example: Command without argument + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { blockquoteSchema } from "@milkdown/kit/preset/commonmark"; +import { wrapIn } from "@milkdown/kit/prose/commands"; +import { $command, callCommand } from "@milkdown/kit/utils"; + +const wrapInBlockquoteCommand = $command( + "WrapInBlockquote", + (ctx) => () => wrapIn(blockquoteSchema.type(ctx)), +); + +// register the command when creating the editor +const editor = Editor().make().use(wrapInBlockquoteCommand).create(); + +// call command +editor.action(callCommand(wrapInBlockquoteCommand.key)); +``` + +### Example: Command with argument + +Commands can accept arguments of any type: + +```typescript +import { headingSchema } from "@milkdown/kit/preset/commonmark"; +import { setBlockType } from "@milkdown/kit/prose/commands"; +import { $command, callCommand } from "@milkdown/kit/utils"; + +// use number as the type of argument +export const WrapInHeading = createCmdKey<number>(); +const wrapInHeadingCommand = $command( + "WrapInHeading", + (ctx) => + (level = 1) => + setBlockType(headingSchema.type(ctx), { level }), +); + +// call command +editor.action(callCommand(wrapInHeadingCommand.key)); // turn to h1 by default +editor.action(callCommand(wrapInHeadingCommand.key, 2)); // turn to h2 +``` + +### Example: Command with Multiple Arguments + +```typescript +interface TableConfig { + rows: number; + cols: number; + withHeader: boolean; +} + +const insertTableCommand = $command( + "InsertTable", + (ctx) => (config: TableConfig) => { + // Implementation for inserting a table + return (state, dispatch) => { + // ... table insertion logic + return true; + }; + }, +); + +// Usage +editor.action( + callCommand(insertTableCommand.key, { + rows: 3, + cols: 3, + withHeader: true, + }), +); +``` + +## Best Practices + +--- + +1. **Command Naming** + - Use clear, descriptive names + - Follow the pattern: `[Action][Target]Command` + - Example: `toggleEmphasisCommand`, `insertTableCommand` + +2. **Command Organization** + - Group related commands together + - Use namespaces for command keys + - Keep commands focused and single-purpose + +3. **Error Handling** + - Always check if the command can be executed + - Return `false` if the command cannot be executed + - Handle edge cases gracefully + +4. **Performance** + - Keep commands lightweight + - Avoid unnecessary state updates + - Use command chaining for complex operations + +5. **Type Safety** + - Use TypeScript for command arguments + - Define clear interfaces for command payloads + - Use generics for type-safe command keys + +## Common Patterns + +--- + +### Toggle Commands + +```typescript +const toggleCommand = $command( + "ToggleFeature", + (ctx) => () => (state, dispatch) => { + const isActive = checkIfActive(state); + return isActive + ? removeFeature(state, dispatch) + : addFeature(state, dispatch); + }, +); +``` + +### Insert Commands + +```typescript +const insertCommand = $command( + "InsertContent", + (ctx) => (content: string) => (state, dispatch) => { + const { selection } = state; + if (!selection) return false; + + const tr = state.tr.insertText(content, selection.from); + dispatch?.(tr); + return true; + }, +); +``` + +### Transform Commands + +```typescript +const transformCommand = $command( + "TransformContent", + (ctx) => (transform: (node: ProseNode) => ProseNode) => (state, dispatch) => { + const { selection } = state; + if (!selection) return false; + + const tr = state.tr.replaceWith( + selection.from, + selection.to, + transform(state.doc.nodeAt(selection.from)!), + ); + dispatch?.(tr); + return true; + }, +); +``` diff --git a/milkdown-docs/guide/faq.md b/milkdown-docs/guide/faq.md new file mode 100644 index 0000000..41e6c7e --- /dev/null +++ b/milkdown-docs/guide/faq.md @@ -0,0 +1,39 @@ +# FAQ + +This page lists answers of FAQ. + +--- + +### How can I change contents programmatically? + +You should use `editor.action` to change the contents. +We provide two macros for that allow you to change content in milkdown, `insert` and `replaceAll`. + +```typescript +import { insert, replaceAll } from "@milkdown/kit/utils"; + +const editor = await Editor.make() + // .use(<All Your Plugins>) + .create(); + +editor.action(insert("# New Heading")); + +editor.action(replaceAll("# New Document")); +``` + +--- + +### How to configure remark? + +```typescript +import { remarkStringifyOptionsCtx } from "@milkdown/kit/core"; + +editor.config((ctx) => { + ctx.set(remarkStringifyOptionsCtx, { + // some options, for example: + bullet: "*", + fences: true, + incrementListMarker: false, + }); +}); +``` diff --git a/milkdown-docs/guide/getting-started.md b/milkdown-docs/guide/getting-started.md new file mode 100644 index 0000000..bb3ef4e --- /dev/null +++ b/milkdown-docs/guide/getting-started.md @@ -0,0 +1,165 @@ +# Getting Started with Milkdown + +Milkdown is a powerful WYSIWYG markdown editor that combines the simplicity of markdown with the flexibility of a modern editor. It's designed to be lightweight yet extensible, making it perfect for both simple and complex editing needs. + +## Quick Start + +The fastest way to get started is using `@milkdown/crepe`: + +```bash +npm install @milkdown/crepe +``` + +```typescript +import { Crepe } from "@milkdown/crepe"; +import "@milkdown/crepe/theme/common/style.css"; +import "@milkdown/crepe/theme/frame.css"; + +const crepe = new Crepe({ + root: "#app", + defaultValue: "Hello, Milkdown!", +}); + +crepe.create(); +``` + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/editor-crepe"} + +## Core Concepts + +Milkdown consists of two main parts: + +1. **Core Package** (`@milkdown/core`) + - Plugin loader + - Internal plugins + +2. **Additional Plugins** + - Syntax support + - Commands + - UI components + - Custom features + +This modular architecture allows you to enable or disable features as needed, from basic markdown support to advanced features like tables, LaTeX equations, and collaborative editing. + +## Key Features + +- ๐Ÿ“ **WYSIWYG Markdown** - Write markdown in an elegant way +- ๐ŸŽจ **Themable** - Create your own theme and publish it as an npm package +- ๐ŸŽฎ **Hackable** - Create your own plugin to support your awesome idea +- ๐Ÿฆพ **Reliable** - Built on top of [prosemirror](https://prosemirror.net/) and [remark](https://github.com/remarkjs/remark) +- โšก **Slash & Tooltip** - Write faster than ever, enabled by a plugin +- ๐Ÿงฎ **Math** - LaTeX math equations support via math plugin +- ๐Ÿ“Š **Table** - Table support with fluent ui, via table plugin +- ๐Ÿป **Collaborate** - Shared editing support with [yjs](https://docs.yjs.dev/) +- ๐Ÿ’พ **Clipboard** - Support copy and paste markdown, via clipboard plugin +- ๐Ÿ‘ **Emoji** - Support emoji shortcut and picker, via emoji plugin + +## Tech Stack + +Milkdown is built on top of these powerful libraries: + +- [Prosemirror](https://prosemirror.net/) - A toolkit for building rich-text editors on the web +- [Remark](https://github.com/remarkjs/remark) - Markdown parser done right +- [TypeScript](https://www.typescriptlang.org/) - For type safety and better developer experience + +## Creating Your First Editor + +Milkdown provides two distinct approaches to create an editor, each suited for different needs: + +### 1. ๐Ÿผ Using `@milkdown/kit` (Build from Scratch) + +This approach gives you complete control over your editor. Use this if you want to: + +- Build a custom editor from the ground up +- Have full control over which features to include +- Create a highly customized editing experience +- Integrate with specific frameworks or requirements + +First, install the required packages: + +```bash +npm install @milkdown/kit +``` + +Create a basic editor with commonmark syntax: + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +// This is the must have css for prosemirror +import "@milkdown/kit/prose/view/style/prosemirror.css"; + +Editor.make().use(commonmark).create(); +``` + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vanilla-commonmark"} + +Add undo & redo support: + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { history } from "@milkdown/kit/plugin/history"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { nord } from "@milkdown/theme-nord"; +import "@milkdown/theme-nord/style.css"; + +const milkdown = Editor.make() + .config(nord) + .use(commonmark) + .use(history) + .create() + .then(() => { + console.log("Editor created"); + }); + +// To destroy the editor +milkdown.destroy(); +``` + +> **Note**: `<Mod>` is `<Cmd>` for macOS and `<Ctrl>` for other platforms. + +### 2. ๐Ÿฅž Using `@milkdown/crepe` (Ready to Use) + +This is the quickest way to get started with a fully-featured editor. Use this if you want to: + +- Get up and running quickly +- Have a well-designed editor out of the box +- Focus on content rather than configuration +- Have a production-ready solution with minimal setup + +```bash +npm install @milkdown/crepe +``` + +```typescript +import { Crepe } from "@milkdown/crepe"; +import "@milkdown/crepe/theme/common/style.css"; +/** + * Available themes: + * frame, classic, nord + * frame-dark, classic-dark, nord-dark + */ +import "@milkdown/crepe/theme/frame.css"; + +const crepe = new Crepe({ + root: "#app", + defaultValue: "Hello, Milkdown!", +}); + +crepe.create().then(() => { + console.log("Editor created"); +}); + +// To destroy the editor +crepe.destroy(); +``` + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/editor-crepe"} + +## Next Steps + +- Learn more about [overview](/guide/architecture-overview) +- Explore [available plugins](/plugins/using-plugins) +- Check out [theming](/guide/theming) + +> ๐Ÿผ Fun fact: This documentation is rendered by Milkdown itself! diff --git a/milkdown-docs/guide/interacting-with-editor.md b/milkdown-docs/guide/interacting-with-editor.md new file mode 100644 index 0000000..45bde14 --- /dev/null +++ b/milkdown-docs/guide/interacting-with-editor.md @@ -0,0 +1,398 @@ +# Interacting with Editor + +This guide covers the essential ways to interact with the Milkdown editor, including initialization, content management, and editor lifecycle. + +## Using Crepe Editor + +--- + +Crepe is a high-level wrapper around Milkdown that provides a simpler API for common editor operations. Here's how to use it: + +```typescript +import { Crepe } from "@milkdown/crepe"; + +// Create a new editor instance +const editor = new Crepe({ + // Optional: specify root element (DOM node or selector) + root: "#editor", + + // Optional: set default content, supports markdown, json and dom. + defaultValue: "# Hello Crepe!", +}); + +// Create the editor +await editor.create(); + +// Get markdown content +const markdown = editor.getMarkdown(); + +// Set readonly mode +editor.setReadonly(true); + +// Register event listeners +editor.on((listener) => { + listener.markdownUpdated((ctx, markdown) => { + console.log("Content updated:", markdown); + }); + + listener.focus((ctx) => { + console.log("Editor focused"); + }); + + listener.blur((ctx) => { + console.log("Editor blurred"); + }); + + listener.selectionUpdated((ctx, selection, prevSelection) => { + console.log("Selection updated:", selection); + }); + + listener.updated((ctx, doc, prevDoc) => { + console.log("Document updated:", doc); + }); +}); + +// Destroy the editor when done +await editor.destroy(); +``` + +## Register to DOM + +--- + +By default, milkdown will create editor on the `document.body`. Alternatively, you can also point out which dom node you want it to load into: + +```typescript +import { rootCtx } from "@milkdown/kit/core"; + +Editor.make().config((ctx) => { + ctx.set(rootCtx, document.querySelector("#editor")); +}); +``` + +It's also possible to just pass a selector to `rootCtx`: + +> The selector will be passed to `document.querySelector` to get the dom. + +```typescript +import { rootCtx } from "@milkdown/kit/core"; + +Editor.make().config((ctx) => { + ctx.set(rootCtx, "#editor"); +}); +``` + +## Setting Default Value + +--- + +We support three types of default values: + +- Markdown strings +- HTML DOM +- Prosemirror documentation JSON + +### Markdown + +You can set a markdown string as the default value of the editor. + +```typescript +import { defaultValueCtx } from "@milkdown/kit/core"; + +const defaultValue = "# Hello milkdown"; +Editor.make().config((ctx) => { + ctx.set(defaultValueCtx, defaultValue); +}); +``` + +### Dom + +You can also use HTML as default value. + +Let's assume that we have the following html snippets: + +```html +<div id="pre"> + <h1>Hello milkdown!</h1> +</div> +``` + +Then we can use it as a defaultValue with a `type` specification: + +```typescript +import { defaultValueCtx } from "@milkdown/kit/core"; + +const defaultValue = { + type: "html", + dom: document.querySelector("#pre"), +}; +Editor.make().config((ctx) => { + ctx.set(defaultValueCtx, defaultValue); +}); +``` + +### JSON + +We can also use a JSON object as a default value. + +This JSON object can be obtained by a listener through the [listener-plugin](https://www.npmjs.com/package/@milkdown/plugin-listener), for example: + +```typescript +import { listener, listenerCtx } from "@milkdown/kit/plugin/listener"; + +let jsonOutput; + +Editor.make() + .config((ctx) => { + ctx.get(listenerCtx).updated((ctx, doc, prevDoc) => { + jsonOutput = doc.toJSON(); + }); + }) + .use(listener); +``` + +Then we can use this `jsonOutput` as default Value: + +```typescript +import { defaultValueCtx } from "@milkdown/kit/core"; + +const defaultValue = { + type: "json", + value: jsonOutput, +}; +Editor.make().config((ctx) => { + ctx.set(defaultValueCtx, defaultValue); +}); +``` + +## Inspecting Editor Status + +--- + +You can inspect the editor's status through the `status` property. + +```typescript +import { Editor, EditorStatus } from "@milkdown/kit/core"; + +const editor = Editor.make().use(/* some plugins */); + +assert(editor.status === EditorStatus.Idle); + +editor.create().then(() => { + assert(editor.status === EditorStatus.Created); +}); + +assert(editor.status === EditorStatus.OnCreate); + +editor.destroy().then(() => { + assert(editor.status === EditorStatus.Destroyed); +}); + +assert(editor.status === EditorStatus.OnDestroyed); +``` + +You can also listen to the status changes: + +```typescript +import { Editor, EditorStatus } from "@milkdown/kit/core"; + +const editor = Editor.make().use(/* some plugins */); + +editor.onStatusChange((status: EditorStatus) => { + console.log(status); +}); +``` + +### Status Lifecycle + +1. `Idle`: Initial state +2. `OnCreate`: During creation +3. `Created`: Successfully created +4. `OnDestroyed`: During destruction +5. `Destroyed`: Successfully destroyed + +## Adding Listeners + +--- + +As mentioned above, you can add a listener to the editor, in order to get its value when needed. +You can add as many listeners as you want, all the listeners will be triggered at once. + +### Markdown Listener + +You can add markdown listener to get the editor's contents as a markdown string. + +> โš ๏ธ Markdown listener will influence the performance for large documents, please use it carefully. +> If you have a large document, I suggest you to only `parse` and `serialize` the document when needed. + +```typescript +import { listener, listenerCtx } from "@milkdown/kit/plugin/listener"; + +let output = ""; + +Editor.make() + .config((ctx) => { + ctx.get(listenerCtx).markdownUpdated((ctx, markdown, prevMarkdown) => { + output = markdown; + }); + }) + .use(listener); +``` + +### Doc Listener + +You can also listen to the [raw prosemirror document node](https://prosemirror.net/docs/ref/#model.Node), and do things you want from there. + +```typescript +import { listener, listenerCtx } from "@milkdown/kit/plugin/listener"; + +let jsonOutput; + +Editor.make() + .config((ctx) => { + ctx.get(listenerCtx).updated((ctx, doc, prevDoc) => { + jsonOutput = doc.toJSON(); + }); + }) + .use(listener); +``` + +### Selection Listener + +You can track changes to the editor's selection using the `selectionUpdated` event. This is useful for implementing features like: + +- Custom toolbars that update based on selection +- Context menus +- Selection-based formatting controls + +```typescript +import { listener, listenerCtx } from "@milkdown/kit/plugin/listener"; +import { Selection, TextSelection } from "@milkdown/prose/state"; + +Editor.make() + .config((ctx) => { + ctx.get(listenerCtx).selectionUpdated((ctx, selection, prevSelection) => { + if (selection instanceof TextSelection) { + // Get selection range + const { from, to } = selection; + + // Example: Update toolbar based on selection + updateToolbar({ + hasSelection: from !== to, + selectionStart: from, + selectionEnd: to, + }); + } + }); + }) + .use(listener); +``` + +The selection listener will be triggered when the selection is changed. +So you don't need to compare them manually. + +For more details about listeners, please check [Using Listeners](/docs/api/plugin-listener). + +## Readonly Mode + +--- + +You can set the editor to readonly mode by setting the `editable` property. + +```typescript +import { editorViewOptionsCtx } from "@milkdown/kit/core"; + +let readonly = false; + +const editable = () => !readonly; + +Editor.make().config((ctx) => { + ctx.update(editorViewOptionsCtx, (prev) => ({ + ...prev, + editable, + })); +}); + +// set to readonly after 5 secs. +setTimeout(() => { + readonly = true; +}, 5000); +``` + +### Use Cases for Readonly Mode + +- Preview mode +- Document review +- Print-friendly views +- Mobile device optimization + +## Using Actions + +--- + +You can use an action to get the context value in a running editor on demand. + +For example, to get the markdown string by running an action: + +```typescript +import { Editor, editorViewCtx, serializerCtx } from "@milkdown/kit/core"; + +async function playWithEditor() { + const editor = await Editor.make().use(commonmark).create(); + + const getMarkdown = () => + editor.action((ctx) => { + const editorView = ctx.get(editorViewCtx); + const serializer = ctx.get(serializerCtx); + return serializer(editorView.state.doc); + }); + + // get markdown string: + getMarkdown(); +} +``` + +We provide some macros out of the box, you can use them as actions: + +```typescript +import { insert } from "@milkdown/kit/utils"; + +editor.action(insert("# Hello milkdown")); +``` + +### Common Actions + +- Insert content +- Get current selection +- Apply formatting +- Execute commands + +For more details about macros, please check [macros](/docs/guide/macros). + +## Destroying + +--- + +You can call `editor.destroy` to destroy an existing editor. You can create a new editor again with `editor.create`. + +```typescript +await editor.destroy(); + +// Then create again +await editor.create(); +``` + +If you just want to recreate the editor, you can use `editor.create`, it will **destroy the old editor and create a new one**. + +```typescript +await editor.create(); + +// This equals to call `editor.destroy` and `editor.create` again. +await editor.create(); +``` + +If you want to **clear the plugins and configs for the editor** when calling `editor.destroy`, you can pass `true` to `editor.destroy`. + +```typescript +await editor.destroy(true); +``` diff --git a/milkdown-docs/guide/keyboard-shortcuts.md b/milkdown-docs/guide/keyboard-shortcuts.md new file mode 100644 index 0000000..96f9bfe --- /dev/null +++ b/milkdown-docs/guide/keyboard-shortcuts.md @@ -0,0 +1,252 @@ +# Keyboard Shortcuts + +Keyboard shortcuts are a crucial part of the editor's user experience. Milkdown provides a flexible system for configuring keyboard shortcuts through presets and plugins. + +## Default Shortcuts + +--- + +Milkdown comes with a set of default keyboard shortcuts from both presets and plugins. Here's a comprehensive list of all internal shortcuts: + +> #### ๐Ÿ’ก Note +> +> `Mod` represents the platform-specific modifier key: +> +> - Windows/Linux: `Ctrl` +> - macOS: `Command` + +### Commonmark Preset Shortcuts + +#### Headings + +| Shortcut | Description | +| -------------------- | ----------------------- | +| `Mod-Alt-1` | Turn block into h1 | +| `Mod-Alt-2` | Turn block into h2 | +| `Mod-Alt-3` | Turn block into h3 | +| `Mod-Alt-4` | Turn block into h4 | +| `Mod-Alt-5` | Turn block into h5 | +| `Mod-Alt-6` | Turn block into h6 | +| `Delete`/`Backspace` | Downgrade heading level | + +#### Block Elements + +| Shortcut | Description | +| ------------- | ---------------------------- | +| `Mod-Shift-b` | Wrap selection in blockquote | +| `Mod-Shift-8` | Wrap in bullet list | +| `Mod-Shift-7` | Wrap in ordered list | +| `Mod-Shift-c` | Wrap in code block | +| `Shift-Enter` | Insert hard break | +| `Mod-Alt-0` | Wrap in paragraph | + +#### Text Formatting + +| Shortcut | Description | +| -------- | ------------------ | +| `Mod-b` | Toggle bold | +| `Mod-i` | Toggle italic | +| `Mod-e` | Toggle inline code | + +### GFM Preset Shortcuts + +#### Text Formatting + +| Shortcut | Description | +| ----------- | -------------------- | +| `Mod-Alt-x` | Toggle strikethrough | + +#### Tables + +| Shortcut | Description | +| ------------------- | -------------------------------- | +| `Mod-]` | Move to next cell | +| `Mod-[` | Move to previous cell | +| `Mod-Enter`/`Enter` | Exit table and break if possible | + +## Configuring Shortcuts + +--- + +You can customize keyboard shortcuts by configuring the keymap in the editor setup: + +```typescript +import { blockquoteKeymap, commonmark } from "@milkdown/kit/preset/commonmark"; + +Editor.make() + .config((ctx) => { + ctx.set(blockquoteKeymap.key, { + WrapInBlockquote: "Mod-Shift-b", + // or you may want to bind multiple keys: + WrapInBlockquote: ["Mod-Shift-b", "Mod-b"], + }); + }) + .use(commonmark); +``` + +## Defining Keymaps + +--- + +Keymaps in Milkdown are defined using the `$useKeymap` utility. Here's how to define keymaps for different features: + +### Heading Keymap Example + +```typescript +import { $useKeymap } from "@milkdown/utils"; +import { commandsCtx } from "@milkdown/core"; + +export const headingKeymap = $useKeymap("headingKeymap", { + TurnIntoH1: { + shortcuts: "Mod-Alt-1", + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(wrapInHeadingCommand.key, 1); + }, + }, + TurnIntoH2: { + shortcuts: "Mod-Alt-2", + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(wrapInHeadingCommand.key, 2); + }, + }, + // ... more heading levels + DowngradeHeading: { + shortcuts: ["Delete", "Backspace"], + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(downgradeHeadingCommand.key); + }, + }, +}); +``` + +### Strong (Bold) Keymap Example + +```typescript +import { $useKeymap } from "@milkdown/utils"; +import { commandsCtx } from "@milkdown/core"; + +export const strongKeymap = $useKeymap("strongKeymap", { + ToggleBold: { + shortcuts: ["Mod-b"], + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(toggleStrongCommand.key); + }, + }, +}); +``` + +### Keymap Structure + +Each keymap definition follows this structure: + +```typescript +$useKeymap('keymapName', { + CommandName: { + shortcuts: string | string[], // Single shortcut or array of shortcuts + priority?: number, // (Optional) Priority of the shortcut + command: (ctx) => () => { // Command to execute + const commands = ctx.get(commandsCtx); + return () => commands.call(commandKey, ...args); + }, + }, +}); +``` + +## Creating Custom Shortcuts + +--- + +If you need to add custom shortcuts, you can create a keymap plugin: + +```typescript +import { $useKeymap } from "@milkdown/utils"; +import { commandsCtx } from "@milkdown/core"; + +const customKeymap = $useKeymap("customKeymap", { + CustomCommand: { + shortcuts: "F1", + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(someCommand.key); + }, + }, +}); + +// Usage +Editor.make().use(customKeymap).use(commonmark); +``` + +### Example: Custom Command with Shortcut + +```typescript +import { $command, $useKeymap } from "@milkdown/utils"; +import { commandsCtx } from "@milkdown/core"; + +// Create a custom command +const customCommand = $command("CustomCommand", (ctx) => () => { + return (state, dispatch) => { + // Command implementation + return true; + }; +}); + +// Create a keymap +const customKeymap = $useKeymap("customKeymap", { + CustomCommand: { + shortcuts: ["F1", "Mod-F1"], // Multiple shortcuts + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(customCommand.key); + }, + }, +}); + +// Usage +Editor.make().use(customCommand).use(customKeymap); +``` + +## Shortcut Priority + +You can control the order in which shortcuts are handled by specifying a `priority` property. Shortcuts with higher priority values are handled before those with lower values. This is useful if you want your custom shortcut to override or take precedence over other shortcuts that use the same key combination. + +When multiple shortcuts are registered for the same key, they are executed in order of priority. If a shortcut command returns `false`, the next shortcut with the same key will be tried. If it returns `true`, no further commands for that key will be run. This allows you to chain or override shortcut behaviors as needed. + +- The default priority is **50**. +- Normal priority values should be between **1** and **100**. +- Use higher numbers to ensure your shortcut is registered before others with the same key. + +#### Example: Using Priority + +```typescript +import { $useKeymap } from "@milkdown/utils"; +import { commandsCtx } from "@milkdown/core"; + +export const customKeymap = $useKeymap("customKeymap", { + CustomBold: { + shortcuts: "Mod-b", + priority: 100, // Highest in the normal range, so this runs first + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => { + // Custom bold logic + return true; + }; + }, + }, + CustomAnotherBold: { + shortcuts: "Mod-b", + priority: 75, // Lower priority, will run only if CustomBold returns false + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => { + // Custom italic logic + return true; + }; + }, + }, +}); +``` diff --git a/milkdown-docs/guide/macros.md b/milkdown-docs/guide/macros.md new file mode 100644 index 0000000..7a2c05e --- /dev/null +++ b/milkdown-docs/guide/macros.md @@ -0,0 +1,278 @@ +# Macros + +Macros are helper functions that provide a convenient way to interact with the editor. They take a payload (or nothing) as parameters and return a callback function that takes the `ctx` of milkdown as a parameter. When called with `ctx`, they apply the specified action to the editor. + +## Usage + +There are two main ways to use macros: + +```typescript +import { insert } from "@milkdown/kit/utils"; +import { listenerCtx } from "@milkdown/plugin-listener"; + +// Method 1: Using editor.action() +editor.action(insert("# Hello Macro")); + +// Method 2: Using listener +editor.config((ctx) => { + ctx.get(listenerCtx).mounted(insert("# Default Title")); +}); +``` + +## Available Macros + +### Content Manipulation + +#### `insert` + +Inserts content at the current cursor position. The macro accepts two parameters: + +- `markdown`: The markdown string to insert +- `inline`: Optional boolean flag (default: false) that determines how the content is inserted + +```typescript +import { insert } from "@milkdown/kit/utils"; + +// Insert as block content (default) +editor.action(insert("# Hello World")); + +// Insert as inline content +editor.action(insert("inline text", true)); +``` + +The behavior differs based on the `inline` parameter: + +- When `inline` is `false` (default): + - Replaces the current selection with the parsed markdown content + - Maintains the selection's open start/end positions + - Scrolls the view to show the inserted content +- When `inline` is `true`: + - Attempts to insert the content as inline text + - If the content is text-only, replaces the selection with a text node + - Otherwise, replaces the selection with the parsed content + +#### `insertPos` + +Inserts markdown at a given position. The macro accepts two parameters: + +- `markdown`: The markdown string to insert +- `pos`: The position to insert the content at + +```typescript +import { insertPos } from "@milkdown/kit/utils"; + +// Insert "Hello" at the beginning of the document +editor.action(insertPos("Hello", 0)); +``` + +#### `replaceAll` + +Replaces all content in the editor. The macro accepts two parameters: + +- `markdown`: The markdown string to replace the current content with +- `flush`: Optional boolean flag (default: false) that determines how the replacement is performed + +```typescript +import { replaceAll } from "@milkdown/kit/utils"; + +// Replace content without flushing state +editor.action(replaceAll("# New Content")); + +// Replace content and flush editor state +editor.action(replaceAll("# New Content", true)); +``` + +The behavior differs based on the `flush` parameter: + +- When `flush` is `false` (default): + - Replaces the entire document content with the new markdown + - Maintains the current editor state + - More efficient for simple content replacements +- When `flush` is `true`: + - Creates a new editor state with the new content + - Reinitializes all plugins + - Useful when you need a completely fresh editor state + +#### `replaceRange` + +Replaces the content of the given range with a markdown string. + +```typescript +import { replaceRange } from "@milkdown/kit/utils"; + +// Replace content from position 0 to 5 with "Hello" +editor.action(replaceRange("Hello", { from: 0, to: 5 })); +``` + +### Content Retrieval + +#### `getMarkdown` + +Gets the current content as markdown. If a range is provided, it will return the markdown for that range; otherwise, it will return the markdown for the entire document. + +```typescript +import { getMarkdown } from "@milkdown/kit/utils"; + +// Get markdown for the entire document +const markdown = editor.action(getMarkdown()); + +// Get markdown for a specific range +const selectionMarkdown = editor.action(getMarkdown({ from: 0, to: 5 })); +``` + +#### `getHTML` + +Gets the current content as HTML. + +```typescript +import { getHTML } from "@milkdown/kit/utils"; + +const html = editor.action(getHTML()); +``` + +### Editor State + +#### `forceUpdate` + +Forces the editor to update its state. + +```typescript +import { forceUpdate } from "@milkdown/kit/utils"; + +editor.action(forceUpdate()); +``` + +#### `setAttr` + +Sets attributes for a node at a specific position. The macro accepts two parameters: + +- `pos`: The position of the node to update +- `update`: A function that takes the previous attributes and returns the new attributes + +```typescript +import { setAttr } from "@milkdown/kit/utils"; + +// Update node attributes at position 10 +editor.action( + setAttr(10, (prevAttrs) => ({ + ...prevAttrs, + class: "custom-class", + })), +); + +// Example: Update heading level +editor.action( + setAttr(10, (prevAttrs) => ({ + ...prevAttrs, + level: 2, + })), +); +``` + +The macro: + +- Takes a specific position in the document +- Retrieves the node at that position +- Applies the update function to modify the node's attributes +- Dispatches the changes to update the editor state + +Note: The position must be valid and contain a node, otherwise the operation will be ignored. + +### Navigation + +#### `outline` + +Gets the outline of the document. + +```typescript +import { outline } from "@milkdown/kit/utils"; + +const docOutline = editor.action(outline()); +``` + +### Command Execution + +#### `callCommand` + +Calls a registered command with optional payload. The macro has two overloads: + +Examples: + +```typescript +import { callCommand } from "@milkdown/kit/utils"; +import { wrapInHeadingCommand } from "@milkdown/plugin-heading"; + +// Using command key +editor.action(callCommand(wrapInHeadingCommand.key, 1)); + +// With complex payload +editor.action( + callCommand("CustomCommand", { + type: "heading", + level: 1, + content: "New Heading", + }), +); +``` + +The macro: + +- Takes a command key +- Optionally accepts a payload parameter +- Returns a boolean indicating whether the command was successful + +Note: The command must be registered in the editor's command context before it can be called. + +### Utility Macros + +#### `markdownToSlice` + +Converts a markdown string to a [slice](https://prosemirror.net/docs/ref/#model.Slice). This is useful when you need to manipulate the content before inserting it into the editor. + +```typescript +import { markdownToSlice } from "@milkdown/kit/utils"; + +const slice = editor.action(markdownToSlice("# Hello Slice")); +``` + +## Examples + +### Adding Content + +```typescript +import { insert } from "@milkdown/kit/utils"; +import { listenerCtx } from "@milkdown/plugin-listener"; + +editor.config((ctx) => { + ctx.get(listenerCtx).mounted(insert("# Welcome\nStart editing...")); +}); +``` + +### Saving Content + +```typescript +import { getMarkdown } from "@milkdown/kit/utils"; + +editor.config((ctx) => { + ctx.get(listenerCtx).updated(() => { + const content = getMarkdown()(ctx); + localStorage.setItem("editor-content", content); + }); +}); +``` + +### Custom Command with Macro + +```typescript +import { callCommand } from "@milkdown/kit/utils"; + +editor.action( + callCommand("customCommand", { + type: "heading", + level: 1, + content: "New Heading", + }), +); +``` + +For more details about each macro's parameters and return types, check the [API Reference](/docs/api/utils#macros). diff --git a/milkdown-docs/guide/prosemirror-api.md b/milkdown-docs/guide/prosemirror-api.md new file mode 100644 index 0000000..4ae104b --- /dev/null +++ b/milkdown-docs/guide/prosemirror-api.md @@ -0,0 +1,38 @@ +# Prosemirror API + +Milkdown is built on top of prosemirror. Which means you can use the entire prosemirror API in Milkdown. +To access the prosemirror API, you can use the `@milkdown/prose` package. It re-exports all of the prosemirror API. +Using this package you can make sure that you are using the same version of prosemirror as Milkdown. + +## Installation + +To access a certain API in the `prosemirror-x` package, you need to import them from `@milkdown/kit/prose/x`. + +For example: + +```ts +// Originally in prosemirror-state +import { EditorState } from "@milkdown/kit/prose/state"; +// Originally in prosemirror-view +import { EditorView } from "@milkdown/kit/prose/view"; +``` + +## List of packages + +The following is a list of all the re-exported prosemirror API. + +- `@milkdown/kit/prose/changeset` +- `@milkdown/kit/prose/commands` +- `@milkdown/kit/prose/dropcursor` +- `@milkdown/kit/prose/gapcursor` +- `@milkdown/kit/prose/history` +- `@milkdown/kit/prose/inputrules` +- `@milkdown/kit/prose/keymap` +- `@milkdown/kit/prose/model` +- `@milkdown/kit/prose/schema-list` +- `@milkdown/kit/prose/state` +- `@milkdown/kit/prose/transform` +- `@milkdown/kit/prose/view` +- `@milkdown/kit/prose/tables` + +You can find the documentation of the prosemirror API [here](https://prosemirror.net/docs/ref/). diff --git a/milkdown-docs/guide/styling.md b/milkdown-docs/guide/styling.md new file mode 100644 index 0000000..f0b7b48 --- /dev/null +++ b/milkdown-docs/guide/styling.md @@ -0,0 +1,215 @@ +# Styling Guide + +Milkdown is a headless editor, which means it doesn't come with any default styles. This gives you complete control over the appearance of your editor. You can either use existing themes or create your own custom styling solution. + +# Styling Crepe Theme + +--- + +Crepe is a collection of themes for Milkdown that provides both light and dark variants. The theme structure is organized as follows: + +``` +theme/ +โ”œโ”€โ”€ common/ # Shared styles and utilities +โ”œโ”€โ”€ crepe/ # Light theme variant +โ”œโ”€โ”€ crepe-dark/ # Dark theme variant +โ”œโ”€โ”€ frame/ # Frame theme (light) +โ”œโ”€โ”€ frame-dark/ # Frame theme (dark) +โ”œโ”€โ”€ nord/ # Nord theme (light) +โ””โ”€โ”€ nord-dark/ # Nord theme (dark) +``` + +## Using Crepe Theme + +To use the Crepe theme in your project: + +```ts +// Import base styles first +import "@milkdown/crepe/theme/common/style.css"; + +// Choose the theme you want to use +import "@milkdown/crepe/theme/crepe.css"; +``` + +## Theme Variables + +Crepe theme uses CSS variables for consistent styling. Here are all the available variables: + +### Colors + +```css +.milkdown { + /* Background Colors */ + --crepe-color-background: #fffdfb; /* Main background color */ + --crepe-color-surface: #fff8f4; /* Surface color for cards/panels */ + --crepe-color-surface-low: #fff1e5; /* Lower surface color for depth */ + + /* Text Colors */ + --crepe-color-on-background: #1f1b16; /* Text color on background */ + --crepe-color-on-surface: #201b13; /* Text color on surface */ + --crepe-color-on-surface-variant: #4f4539; /* Secondary text color */ + + /* Accent Colors */ + --crepe-color-primary: #805610; /* Primary brand color */ + --crepe-color-secondary: #fbdebc; /* Secondary accent color */ + --crepe-color-on-secondary: #271904; /* Text color on secondary */ + + /* UI Colors */ + --crepe-color-outline: #817567; /* Border/outline color */ + --crepe-color-inverse: #362f27; /* Inverse color for contrast */ + --crepe-color-on-inverse: #fcefe2; /* Text color on inverse */ + --crepe-color-inline-code: #ba1a1a; /* Inline code color */ + --crepe-color-error: #ba1a1a; /* Error state color */ + + /* Interactive Colors */ + --crepe-color-hover: #f9ecdf; /* Hover state color */ + --crepe-color-selected: #ede0d4; /* Selected state color */ + --crepe-color-inline-area: #e4d8cc; /* Inline editing area color */ +} +``` + +### Typography + +```css +.milkdown { + /* Font Families */ + --crepe-font-title: Georgia, Cambria, "Times New Roman", Times, serif; + --crepe-font-default: "Open Sans", Arial, Helvetica, sans-serif; + --crepe-font-code: + Fira Code, Menlo, Monaco, "Courier New", Courier, monospace; +} +``` + +### Shadows + +```css +.milkdown { + /* Small Shadow */ + --crepe-shadow-1: + 0px 1px 3px 1px rgba(0, 0, 0, 0.15), 0px 1px 2px 0px rgba(0, 0, 0, 0.3); + /* Large Shadow */ + --crepe-shadow-2: + 0px 2px 6px 2px rgba(0, 0, 0, 0.15), 0px 1px 2px 0px rgba(0, 0, 0, 0.3); +} +``` + +## Customizing Crepe Theme + +You can customize the Crepe theme by overriding its variables: + +```css +/* custom-overrides.css */ +.crepe .milkdown { + /* Override colors */ + --crepe-color-primary: #your-primary-color; + --crepe-color-background: #your-background-color; + + /* Override typography */ + --crepe-font-default: "Your Font", sans-serif; + + /* Override shadows */ + --crepe-shadow-1: your-shadow-value; +} +``` + +# Styling Milkdown + +--- + +## Basic Styling + +The editor is rendered within a container that has the class `.milkdown`, and the editable content area is wrapped in a container with the class `.editor`. You can use these classes to scope your styles: + +```css +/* Basic styling example */ +.milkdown .editor { + max-width: 800px; + margin: 0 auto; + padding: 1rem; +} + +.milkdown .editor p { + margin: 1rem 0; + line-height: 1.6; +} +``` + +## Node and Mark Classes + +Milkdown provides default class names for each node and mark. Here are some common examples: + +```css +/* Paragraph styling */ +.milkdown .editor .paragraph { + margin: 1rem 0; +} + +/* Heading styling */ +.milkdown .editor .heading { + font-weight: 600; + margin: 1.5rem 0 1rem; +} + +/* List styling */ +.milkdown .editor .bullet-list { + padding-left: 1.5rem; +} + +.milkdown .editor .ordered-list { + padding-left: 1.5rem; +} +``` + +## Custom Attributes + +You can add custom attributes to nodes and marks, which is particularly useful when working with CSS frameworks like Tailwind CSS. + +```typescript +import { Editor, editorViewOptionsCtx } from "@milkdown/kit/core"; +import { + commonmark, + headingAttr, + paragraphAttr, +} from "@milkdown/kit/preset/commonmark"; + +Editor.make() + .config((ctx) => { + // Add attributes to the editor container + ctx.update(editorViewOptionsCtx, (prev) => ({ + ...prev, + attributes: { + class: "milkdown-editor mx-auto outline-hidden", + spellcheck: "false", + }, + })); + + // Add attributes to nodes and marks + ctx.set(headingAttr.key, (node) => { + const level = node.attrs.level; + return { + class: `heading-${level} font-bold`, + "data-level": level, + }; + }); + + ctx.set(paragraphAttr.key, () => ({ + class: "text-base leading-relaxed", + })); + }) + .use(commonmark); +``` + +# Best Practices + +--- + +1. **Use CSS Variables**: Define your theme's colors and spacing using CSS variables for easy customization. +2. **Responsive Design**: Ensure your editor styles work well on different screen sizes. +3. **Dark Mode Support**: Consider adding dark mode support using CSS variables and media queries. +4. **Accessibility**: Maintain good contrast ratios and readable font sizes. +5. **Performance**: Keep your CSS selectors specific and avoid overly complex rules. + +For more examples and inspiration, check out: + +- [@milkdown/theme-nord](https://github.com/Milkdown/milkdown/tree/main/packages/theme-nord) +- [@milkdown/crepe/theme](https://github.com/Milkdown/milkdown/tree/main/packages/crepe/src/theme) diff --git a/milkdown-docs/guide/using-crepe.md b/milkdown-docs/guide/using-crepe.md new file mode 100644 index 0000000..870c47b --- /dev/null +++ b/milkdown-docs/guide/using-crepe.md @@ -0,0 +1,234 @@ +# Using Crepe Editor + +Crepe is a powerful, feature-rich Markdown editor built on top of Milkdown. It provides a complete editing experience with a beautiful UI and extensive customization options. + +## Why Choose Crepe? + +--- + +- ๐Ÿš€ **Ready to Use**: Works out of the box with sensible defaults +- ๐ŸŽจ **Beautiful UI**: Modern design with multiple theme options +- ๐Ÿ”ง **Highly Customizable**: Extensive configuration options +- ๐Ÿ“ฆ **Feature Complete**: Includes all essential Markdown editing features +- ๐Ÿ› ๏ธ **Extensible**: Built on Milkdown's plugin system + +## Quick Start + +--- + +### Installation + +```bash +# Using npm +npm install @milkdown/crepe + +# Using yarn +yarn add @milkdown/crepe + +# Using pnpm +pnpm add @milkdown/crepe +``` + +### Basic Usage + +```typescript +import { Crepe } from "@milkdown/crepe"; +import "@milkdown/crepe/theme/common/style.css"; +import "@milkdown/crepe/theme/frame.css"; + +// Choose your preferred theme + +// Create editor instance +const crepe = new Crepe({ + root: document.getElementById("app"), + defaultValue: "# Hello, Crepe!\n\nStart writing your markdown...", +}); + +// Initialize the editor +await crepe.create(); + +// Clean up when done +crepe.destroy(); +``` + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/editor-crepe"} + +## Themes + +--- + +Crepe comes with several beautiful themes out of the box: + +### Light Themes + +- `frame` - Modern frame-based design +- `classic` - Traditional editor look +- `nord` - Clean, minimal Nord color scheme + +### Dark Themes + +- `frame-dark` - Dark version of frame theme +- `classic-dark` - Dark version of classic theme +- `nord-dark` - Dark version of nord theme + +To use a theme: + +```typescript +// Import base styles first +import "@milkdown/crepe/theme/common/style.css"; +// Then import your chosen theme +import "@milkdown/crepe/theme/frame.css"; +``` + +### Custom Themes + +You can create your own theme by extending the base styles. Check out the [existing themes](https://github.com/Milkdown/milkdown/tree/main/packages/crepe/src/theme) for reference. + +## Features + +--- + +Crepe includes a comprehensive set of features that can be enabled or disabled as needed. + +### Feature Configuration + +> **Note**: For any configuration that ends with `Icon` (like `boldIcon`, `linkIcon`, etc.), you can use a HTML string or a simply string. This applies to all icon configurations throughout Crepe's features. + +```typescript +const crepe = new Crepe({ + features: { + // Disable specific features + [Crepe.Feature.CodeMirror]: false, + [Crepe.Feature.Table]: false, + }, + featureConfigs: { + // Configure feature behavior + [Crepe.Feature.LinkTooltip]: { + inputPlaceholder: "Enter URL...", + }, + }, +}); +``` + +### Available Features + +#### 1. Code Editor (`CodeMirror`) + +Syntax highlighting and editing for code blocks with language support, theme customization, and preview capabilities. + +#### 2. List Management (`ListItem`) + +Support for bullet lists, ordered lists, and todo lists with customizable icons and formatting. + +#### 3. Link Management (`LinkTooltip`) + +Enhanced link editing and preview with customizable tooltips, edit/remove actions, and copy functionality. + +#### 4. Image Handling (`ImageBlock`) + +Image upload and management with resizing, captions, and support for both inline and block images. + +#### 5. Block Editing (`BlockEdit`) + +Drag-and-drop block management and slash commands for quick content insertion and organization. + +#### 6. Table Support (`Table`) + +Full-featured table editing with row/column management, alignment options, and drag-and-drop functionality. + +#### 7. Toolbar (`Toolbar`) + +Formatting toolbar for selected text with customizable icons and actions. + +#### 8. Cursor (`Cursor`) + +Enhanced cursor experience with drop cursor and gap cursor for better content placement. + +#### 9. Placeholder (`Placeholder`) + +Document or block level placeholders to guide users when content is empty. + +#### 10. Latex (`Latex`) + +Mathematical formula support with both inline and block math rendering using KaTeX. + +For detailed configuration options of each feature, please refer to the [API documentation](/docs/api/crepe). + +## Editor Instance Methods + +--- + +#### `crepe.editor` + +Access the underlying Milkdown editor instance. + +```typescript +const editor = crepe.editor; +editor.use(customPlugin); +editor.action(insert("Hello")); +``` + +#### `crepe.create()` + +Initialize the editor. + +```typescript +await crepe.create(); +``` + +#### `crepe.destroy()` + +Clean up the editor instance. + +```typescript +crepe.destroy(); +``` + +#### `crepe.setReadonly(value: boolean)` + +Toggle readonly mode. + +```typescript +crepe.setReadonly(true); // Make editor read-only +crepe.setReadonly(false); // Make editor editable +``` + +#### `crepe.on` + +Add event listeners. + +```typescript +crepe.on((listener) => { + listener.markdownUpdated((markdown) => { + console.log("Markdown updated:", markdown); + }); + + listener.updated((doc) => { + console.log("Document updated"); + }); + + listener.focus(() => { + console.log("Editor focused"); + }); + + listener.blur(() => { + console.log("Editor blurred"); + }); +}); +``` + +#### `crepe.getMarkdown()` + +Get current markdown content. + +```typescript +const markdown = crepe.getMarkdown(); +``` + +## Next Steps + +--- + +- Learn about [Milkdown's architecture](/docs/guide/architecture-overview) +- Explore [available plugins](/docs/plugin/using-plugins) +- Read the [API reference](/docs/api/crepe) diff --git a/milkdown-docs/guide/using-milkdown-kit.md b/milkdown-docs/guide/using-milkdown-kit.md new file mode 100644 index 0000000..f4e9d78 --- /dev/null +++ b/milkdown-docs/guide/using-milkdown-kit.md @@ -0,0 +1,37 @@ +# Using @milkdown/kit + +Milkdown provides a set of utilities to help you build your editor. +These utilities are re-exported from the `@milkdown/kit` package. +Thus, you don't need to install the common dependencies manually like `@milkdown/prose`, `@milkdown/core` or `@milkdown/preset-common` in your project. + +## What's included + +`@milkdown/kit` re-exports the following packages: + +| Package | Import path | Scope | +| ---------------------------------------------------------- | ----------------------------------------- | --------- | +| [@milkdown/core](/docs/api/core) | `@milkdown/kit/core` | Framework | +| [@milkdown/ctx](/docs/api/ctx) | `@milkdown/kit/ctx` | Framework | +| [@milkdown/prose](/docs/guide/prosemirror-api) | `@milkdown/kit/prose` | Framework | +| [@milkdown/prose/\*](/docs/guide/prosemirror-api) | `@milkdown/kit/prose/*` | Framework | +| [@milkdown/transformer](/docs/api/transformer) | `@milkdown/kit/transformer` | Framework | +| [@milkdown/utils](/docs/api/utils) | `@milkdown/kit/utils` | Framework | +| [@milkdown/preset-commonmark](/docs/api/preset-commonmark) | `@milkdown/kit/preset/commonmark` | Preset | +| [@milkdown/preset-gfm](/docs/api/preset-gfm) | `@milkdown/kit/preset/gfm` | Preset | +| [@milkdown/plugin-block](/docs/api/plugin-block) | `@milkdown/kit/plugin/block` | Plugin | +| [@milkdown/plugin-clipboard](/docs/api/plugin-clipboard) | `@milkdown/kit/plugin/clipboard` | Plugin | +| [@milkdown/plugin-cursor](/docs/api/plugin-cursor) | `@milkdown/kit/plugin/cursor` | Plugin | +| [@milkdown/plugin-history](/docs/api/plugin-history) | `@milkdown/kit/plugin/history` | Plugin | +| [@milkdown/plugin-indent](/docs/api/plugin-indent) | `@milkdown/kit/plugin/indent` | Plugin | +| [@milkdown/plugin-listener](/docs/api/plugin-listener) | `@milkdown/kit/plugin/listener` | Plugin | +| [@milkdown/plugin-slash](/docs/api/plugin-slash) | `@milkdown/kit/plugin/slash` | Plugin | +| [@milkdown/plugin-tooltip](/docs/api/plugin-tooltip) | `@milkdown/kit/plugin/tooltip` | Plugin | +| [@milkdown/plugin-trailing](/docs/api/plugin-trailing) | `@milkdown/kit/plugin/trailing` | Plugin | +| [@milkdown/plugin-upload](/docs/api/plugin-upload) | `@milkdown/kit/plugin/upload` | Plugin | +| @milkdown/component | `@milkdown/kit/component` | Component | +| @milkdown/component/code-block | `@milkdown/kit/component/code-block` | Component | +| @milkdown/component/image-block | `@milkdown/kit/component/image-block` | Component | +| @milkdown/component/image-inline | `@milkdown/kit/component/image-inline` | Component | +| @milkdown/component/link-tooltip | `@milkdown/kit/component/link-tooltip` | Component | +| @milkdown/component/list-item-block | `@milkdown/kit/component/list-item-block` | Component | +| @milkdown/component/table-block | `@milkdown/kit/component/table-block` | Component | diff --git a/milkdown-docs/guide/why-milkdown.md b/milkdown-docs/guide/why-milkdown.md new file mode 100644 index 0000000..86c269c --- /dev/null +++ b/milkdown-docs/guide/why-milkdown.md @@ -0,0 +1,30 @@ +# Why Milkdown + +There are different kinds of markdown editors, such as [Typora](https://typora.io/), [tui](https://github.com/nhn/tui.editor) and [Bear](https://bear.app/). +They work pretty well for writing notes in markdown on different platforms. So why bother making Milkdown? + +Milkdown aims to provide an **open source solution** for developers to make their editors more powerful, and attractive, it also ensures it runs everywhere. + +--- + +## Open Source & Easy to Integrate + +Different from industrial apps such as [Notion](https://notion.so) and [Typora](https://typora.io/), +Milkdown is open source and fully free. You can integrate it everywhere legally. + +> If you like milkdown, please consider to fund me in order to help with the maintenance. + +## Plugin Driven + +Milkdown treats every feature as a plugin. +With this pattern, developers can choose what they need in an editor instead of bundling all features even they won't need. +Developers can extend their plugins to satisfy their habits such as defining a vim keymap via a custom plugin. + +## Reliable + +Milkdown is powered by [Prosemirror](https://prosemirror.net/) and [Remark](https://github.com/remarkjs/remark), which has a large community and stands the test of the industry. +What's more, plugins from the prosemirror and remark community can be easily reused in order to build a Milkdown plugin. + +## Themable & Hackable + +Themes and plugins for Milkdown can be shared and installed using npm packages. Milkdown is a headless component, which means you can fully control its style. diff --git a/milkdown-docs/playground/template.md b/milkdown-docs/playground/template.md new file mode 100644 index 0000000..e36352e --- /dev/null +++ b/milkdown-docs/playground/template.md @@ -0,0 +1,90 @@ +# Milkdown + +๐Ÿ‘‹ Welcome to Milkdown. We are so glad to see you here! + +๐Ÿ’ญ You may wonder, what is Milkdown? Please write something here. + +> โš ๏ธ **Not the right side!** +> +> Please try something on the left side. + +![1.00](/polar.jpeg "Hello by a polar bear") + +You're seeing this editor called **๐ŸฅžCrepe**, which is an editor built on top of Milkdown. + +If you want to install this editor, you can run `npm install @milkdown/crepe`. Then you can use it like this: + +```js +import { Crepe } from "@milkdown/crepe"; +import "@milkdown/crepe/theme/common/style.css"; +// We have some themes for you to choose, ex. +import "@milkdown/crepe/theme/frame.css"; + +// Or you can create your own theme +import "./your-theme.css"; + +const crepe = new Crepe({ + root: "#app", + defaultValue: "# Hello, Milkdown!", +}); + +crepe.create().then(() => { + console.log("Milkdown is ready!"); +}); + +// Before unmount +crepe.destroy(); +``` + +--- + +## Structure + +> ๐Ÿผ [Milkdown][repo] is a WYSIWYG markdown editor framework. +> +> Which means you can build your own markdown editor with Milkdown. + +In the real world, a typical milkdown editor is built on top of 3 layers: + +- [x] ๐Ÿฅ› Core: The core of Milkdown, which provides the plugin loading system with the editor concepts. +- [x] ๐Ÿง‡ Plugins: A set of plugins that can be used to extend the functionalities of the editor. +- [x] ๐Ÿฎ Components: Some headless components that can be used to build your own editor. + +At the start, you may find it hard to understand all these concepts. +But don't worry, we have this `@milkdown/crepe` editor for you to get started quickly. + +--- + +## You can do more with Milkdown + +In Milkdown, you can extend the editor in many ways: + +| Feature | Description | Example | +| ------------ | ---------------------------------------------------- | ------------------------- | +| ๐ŸŽจ Theme | Create your own theme with CSS | Nord, Dracula | +| ๐Ÿงฉ Plugin | Create your own plugin to extend the editor | Search, Collab | +| ๐Ÿ“ฆ Component | Create your own component to build your own editor | Slash Menu, Toolbar | +| ๐Ÿ“š Syntax | Create your own syntax to extend the markdown parser | Image with Caption, LaTex | + +We have provided a lot of plugins and components, with an out-of-the-box crepe editor for you to use and learn. + +--- + +## Open Source + +- Milkdown is an open-source project under the MIT license. +- Everyone is welcome to contribute to the project, and you can use it in your own project for free. +- Please let me know what you are building with Milkdown, I would be so glad to see that! + +Maintaining Milkdown is a lot of work, and we are working on it in our spare time. +If you like Milkdown, please consider supporting us by [sponsoring][sponsor] the project. +We'll be so grateful for your support. + +## Who built Milkdown? + +Milkdown is built by [Mirone][mirone] and designed by [Meo][meo]. + +[repo]: https://github.com/Milkdown/milkdown +[mirone]: https://github.com/Saul-Mirone +[meo]: https://meo.cool +[sponsor]: https://github.com/sponsors/Saul-Mirone diff --git a/milkdown-docs/plugin/composable-plugins.md b/milkdown-docs/plugin/composable-plugins.md new file mode 100644 index 0000000..4adbfb3 --- /dev/null +++ b/milkdown-docs/plugin/composable-plugins.md @@ -0,0 +1,85 @@ +# Composable Plugins + +In the previous section, we showed you how to create a plugin from scratch. Luckily, you don't need to do that in most cases. Milkdown provides a lot of helpers in [@milkdown/utils](/docs/api/utils) to make it easier to create plugins. The **composable** here means that you can use the plugin in other plugins. For example, you can use a command plugin in a keymap plugin. This is a very common pattern in Milkdown. + +I'll show you some examples of how to use composable plugins. But I won't go into detail about the options and the usage of each plugin. You can find the details in the [API reference](/docs/api/utils#composable). + +## Schema + +The schema plugin is the most important plugin in Milkdown. It defines the structure of the document. A schema plugin in milkdown is a super set of the [node schema spec](https://prosemirror.net/docs/ref/#model.NodeSpec) or [mark schema spec](https://prosemirror.net/docs/ref/#model.MarkSpec) in ProseMirror. + +Let's create a simple blockquote node plugin as an example: + +```typescript +import { $node } from "@milkdown/kit/utils"; + +const blockquote = $node("blockquote", () => ({ + content: "block+", + group: "block", + defining: true, + parseDOM: [{ tag: "blockquote" }], + toDOM: (node) => ["blockquote", ctx.get(blockquoteAttr.key)(node), 0], + parseMarkdown: { + match: ({ type }) => type === "blockquote", + runner: (state, node, type) => { + state.openNode(type).next(node.children).closeNode(); + }, + }, + toMarkdown: { + match: (node) => node.type.name === "blockquote", + runner: (state, node) => { + state.openNode("blockquote").next(node.content).closeNode(); + }, + }, +})); +``` + +## Input Rule + +Since we have a blockquote node, we can create an input rule plugin to make it easier to create a blockquote node. +We expect that when we type `> ` at the beginning of a line, the blockquote node will be created. + +```typescript +import { wrappingInputRule } from "@milkdown/kit/prose/inputrules"; +import { $inputRule } from "@milkdown/kit/utils"; + +export const wrapInBlockquoteInputRule = $inputRule(() => + wrappingInputRule(/^\s*>\s$/, blockquoteSchema.type()), +); +``` + +## Command + +We can also create a command plugin to create a blockquote node. +The command is useful when we want to create a button to create a blockquote node. + +```typescript +import { wrapIn } from "@milkdown/kit/prose/commands"; +import { $command } from "@milkdown/kit/utils"; + +export const wrapInBlockquoteCommand = $command( + "WrapInBlockquote", + () => () => wrapIn(blockquoteSchema.type()), +); +``` + +## Shortcut + +We can also create a shortcut plugin for blockquote. +Here we use `Ctrl + Shift + B` as the shortcut. When we press this shortcut, the blockquote node will be created. +And we can also use the command we created in the previous section. + +```typescript +import { commandsCtx } from "@milkdown/kit/core"; +import { $useKeymap } from "@milkdown/kit/utils"; + +export const blockquoteKeymap = $useKeymap("blockquoteKeymap", { + WrapInBlockquote: { + shortcuts: "Mod-Shift-b", + command: (ctx) => { + const commands = ctx.get(commandsCtx); + return () => commands.call(wrapInBlockquoteCommand.key); + }, + }, +}); +``` diff --git a/milkdown-docs/plugin/example-block-plugin.md b/milkdown-docs/plugin/example-block-plugin.md new file mode 100644 index 0000000..0e506ee --- /dev/null +++ b/milkdown-docs/plugin/example-block-plugin.md @@ -0,0 +1,125 @@ +# Example: Block Plugin + +The **block plugin** adds a positional hook next to every top-level node (paragraphs, headings, lists, etc.). +It is the foundation for features such as drag handles, quick-insert buttons or block toolbars. + +In Milkdown this functionality lives in `@milkdown/plugin-block` and โ€“ consistent with tooltip & slash โ€“ consists of: + +- a **BlockProvider** that deals with DOM positioning/lifecycle +- a **blockFactory** โ€“ _implemented internally_ โ€“ exposed as two ctx slices: `blockSpec`, `blockPlugin` + +This guide covers: + +- Understanding the provider/service architecture. +- Writing a **vanilla TypeScript** drag handle that lets you reorder blocks. +- Mounting custom UIs in **React** and **Vue**. +- Studying the production-ready _Block Handle_ feature inside Crepe. + +--- + +## 1. Anatomy of a Block Plugin + +Unlike tooltip/slash, `@milkdown/plugin-block` ships its factory slices directly: + +```ts +import { blockSpec, blockPlugin } from "@milkdown/plugin-block"; +``` + +You normally interact with **BlockProvider** which talks to an internal _BlockService_: the service listens to mouse / drag events, figures out which node is **active** and sends `show` / `hide` messages to the provider. +Your job is to decide how to render a UI for that active node. + +Key APIs: + +- `new BlockProvider({ ctx, content, ... })` โ€“ similar to Tooltip/Slash. +- `provider.active` โ€“ info about the currently focused block (`node`, `pos`, `el`). +- Optional callbacks: `getOffset`, `getPlacement`, `getPosition` for fine-grained positioning. + +--- + +## 2. Minimal Vanilla Drag Handle + +Below we build a small **drag handle** that appears on hover and lets you drag-n-drop any block. + +```ts +import { block, blockPlugin } from "@milkdown/plugin-block"; +import { BlockProvider } from "@milkdown/plugin-block/block-provider"; // path depending on bundler +import { Editor } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; + +// 1๏ธโƒฃ Create DOM element for the handle +const handle = document.createElement("div"); +handle.className = "drag-handle"; +handle.innerHTML = "โ‰ก"; +handle.style.cssText = ` + width:20px;height:20px;display:flex;align-items:center;justify-content:center; + cursor:grab;border-radius:4px;background:#f2f3f5;color:#555;user-select:none; +`; + +// 2๏ธโƒฃ Build provider โ€“ show only when mouse is over a block +const provider = (ctx: Ctx) => { + const provider = new BlockProvider({ + ctx, + content: handle, + getOffset: () => 8, + }); + + return { + update: provider.update, + destroy: provider.destroy, + }; +}; + +// 3๏ธโƒฃ Wire provider to Milkdown +const blockConfig = (ctx: Ctx) => { + ctx.set(blockSpec.key, { + view: provider(ctx), + }); +}; + +Editor.make().config(blockConfig).use(commonmark).use(block).create(); +``` + +Drag & Drop: + +The HTML element has `cursor:grab`. The internal `BlockService` automatically sets `draggable` and wires ProseMirror's drag-events so you can reorder blocks without extra code ๐Ÿ‘‰ nice! + +--- + +## 3. Framework Examples + +### React + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/react-block"} + +The React demo renders a `<BlockHandle/>` component, keeps drag state in hooks and feeds the root element to `BlockProvider`. + +### Vue + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vue-block"} + +Vue's `<BlockHandle>` uses `Teleport` and reactive refs exactly like the tooltip/slash examples. + +--- + +## 4. Real-world Feature โ€“ Crepe Block Handle + +Crepe brings all the pieces together to create a **block edit** experience that combines a drag handle **and** a plus-button to open the slash menu: + +```text +packages/crepe/src/feature/block-edit/handle/ +``` + +Things worth exploring: + +1. **Dynamic placement** via `getPlacement` (centred vs top-aligned depending on node height). +2. Filtering nodes with `blockConfig.filterNodes` so handles do not appear inside tables / math / blockquotes. +3. Programmatically showing the _slash menu_ after pressing the "+" button. + +--- + +## 5. Summary & Next Steps + +`@milkdown/plugin-block` is the Swiss-army knife for any block-level UI: drag handles, add-buttons, side toolbarsโ€ฆ +Combine it with tooltip/slash to build sophisticated editors. + +Hack on the examples, tweak positioning callbacks, and ship your own block goodies ๐Ÿš€. diff --git a/milkdown-docs/plugin/example-iframe-plugin.md b/milkdown-docs/plugin/example-iframe-plugin.md new file mode 100644 index 0000000..4336c1f --- /dev/null +++ b/milkdown-docs/plugin/example-iframe-plugin.md @@ -0,0 +1,159 @@ +# Example: Iframe Plugin + +This guide demonstrates how to create a custom iframe syntax plugin for Milkdown. This plugin allows you to embed iframes directly in your markdown content using a simple directive syntax. + +## Overview + +--- + +The iframe plugin enables you to embed external web content using the following syntax: + +```markdown +::iframe{src="https://example.com"} +``` + +This will render as an embedded iframe in your document. + +## Implementation Steps + +--- + +To create a custom syntax plugin in Milkdown, we need to implement five key components: + +1. **Remark Plugin**: Parse the custom syntax +2. **Schema Definition**: Define the node structure +3. **Parser**: Convert markdown to ProseMirror nodes +4. **Serializer**: Convert ProseMirror nodes back to markdown +5. **Input Rules**: Handle user input + +Let's implement each component: + +## 1. Remark Plugin + +--- + +First, we use the `remark-directive` plugin to support our custom syntax. This plugin allows us to define custom directives in markdown. + +```typescript +import directive from "remark-directive"; +import { $remark } from "@milkdown/kit/utils"; + +const remarkDirective = $remark("remarkDirective", () => directive); +``` + +## 2. Schema Definition + +--- + +Next, we define the schema for our iframe node. The schema specifies how the node behaves and appears in the editor. + +```typescript +import { $node } from "@milkdown/kit/utils"; +import { Node } from "@milkdown/kit/prose/model"; + +const iframeNode = $node("iframe", () => ({ + group: "block", // Block-level node + atom: true, // Cannot be split + isolating: true, // Cannot be merged with adjacent nodes + marks: "", // No marks allowed + attrs: { + src: { default: null }, // URL attribute + }, + parseDOM: [ + { + tag: "iframe", + getAttrs: (dom) => ({ + src: (dom as HTMLElement).getAttribute("src"), + }), + }, + ], + toDOM: (node: Node) => [ + "iframe", + { ...node.attrs, contenteditable: false }, // Prevent editing iframe content + 0, + ], +})); +``` + +## 3. Parser + +--- + +The parser converts our markdown syntax into ProseMirror nodes. It looks for the `leafDirective` type with the name "iframe". + +```typescript +parseMarkdown: { + match: (node) => node.type === 'leafDirective' && node.name === 'iframe', + runner: (state, node, type) => { + state.addNode(type, { src: (node.attributes as { src: string }).src }); + }, +}, +``` + +## 4. Serializer + +--- + +The serializer converts ProseMirror nodes back to markdown format. + +```typescript +toMarkdown: { + match: (node) => node.type.name === 'iframe', + runner: (state, node) => { + state.addNode('leafDirective', undefined, undefined, { + name: 'iframe', + attributes: { src: node.attrs.src }, + }); + }, +}, +``` + +## 5. Input Rules + +--- + +Input rules handle user typing and convert the syntax into an iframe node. + +```typescript +import { InputRule } from "@milkdown/kit/prose"; +import { $inputRule } from "@milkdown/kit/utils"; + +const iframeInputRule = $inputRule( + () => + new InputRule( + /::iframe\{src\="(?<src>[^"]+)?"?\}/, + (state, match, start, end) => { + const [okay, src = ""] = match; + const { tr } = state; + if (okay) { + tr.replaceWith(start - 1, end, iframeNode.type().create({ src })); + } + return tr; + }, + ), +); +``` + +## Usage + +--- + +To use the iframe plugin, add it to your Milkdown editor configuration: + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; + +Editor.make() + .use([remarkDirective, iframeNode, iframeInputRule]) + .use(commonmark) + .create(); +``` + +## Example + +--- + +Here's a complete example of the iframe plugin in action: + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vanilla-iframe-syntax"} diff --git a/milkdown-docs/plugin/example-marker-plugin.md b/milkdown-docs/plugin/example-marker-plugin.md new file mode 100644 index 0000000..dfc291c --- /dev/null +++ b/milkdown-docs/plugin/example-marker-plugin.md @@ -0,0 +1,189 @@ +# Example: Marker Plugin + +This guide demonstrates how to create a custom marker syntax plugin for Milkdown. This plugin allows you to mark text with custom colors using a simple markdown syntax. + +## Overview + +--- + +The marker plugin enables you to mark text using the following syntax: + +```markdown +==marked text== +=={#EE4B2B}marked text with color== +``` + +This will render as marked text in your document, with the option to specify custom colors. + +## Implementation Steps + +--- + +To create a custom marker syntax plugin in Milkdown, we need to implement several components: + +1. **Remark Plugin**: Parse the custom syntax +2. **Schema Definition**: Define the mark structure +3. **Parser**: Convert markdown to ProseMirror marks +4. **Serializer**: Convert ProseMirror marks back to markdown +5. **Input Rules**: Handle user input +6. **Color Picker**: Add UI for color selection + +Let's implement each component: + +## 1. Remark Plugin + +--- + +First, we create a remark plugin to handle our custom marker syntax: + +> โš ๏ธ The real implementation is more complex, but we simplify it for the sake of the example. +> Under the hood, you'll need to write a [micromark extension](https://github.com/micromark/micromark) to make it works correctly. + +```typescript +import { $remark } from "@milkdown/kit/utils"; + +const remarkMarkColor = () => { + return (tree: any) => { + visit(tree, "text", (node: any, index: number, parent: any) => { + const match = node.value.match(/==(?:{#([^}]+)})?([^=]+)==/); + if (match) { + const [_, color, text] = match; + const mark = { + type: "mark", + data: { color }, + children: [{ type: "text", value: text }], + }; + parent.children.splice(index, 1, mark); + } + }); + }; +}; + +const milkdownMarkColorPlugin = $remark("markColor", () => remarkMarkColor); +``` + +## 2. Schema Definition + +--- + +Next, we define the schema for our marker: + +```typescript +import { $markSchema } from "@milkdown/kit/utils"; +import { Mark } from "mdast"; + +export const DEFAULT_COLOR = "#ffff00"; + +export const markSchema = $markSchema("mark", () => ({ + attrs: { + color: { + default: DEFAULT_COLOR, + validate: "string", + }, + }, + parseDOM: [ + { + tag: "mark", + getAttrs: (node: HTMLElement) => ({ + color: node.style.backgroundColor, + }), + }, + ], + toDOM: (mark) => ["mark", { style: `background-color: ${mark.attrs.color}` }], + parseMarkdown: { + match: (node) => node.type === "mark", + runner: (state, node, markType) => { + const color = (node as Mark).data?.color; + state.openMark(markType, { color }); + state.next(node.children); + state.closeMark(markType); + }, + }, + toMarkdown: { + match: (node) => node.type.name === "mark", + runner: (state, mark) => { + let color = mark.attrs.color; + if (color?.toLowerCase() === DEFAULT_COLOR.toLowerCase()) { + color = undefined; + } + state.withMark(mark, "mark", undefined, { + data: { color }, + }); + }, + }, +})); +``` + +## 3. Input Rules + +--- + +We add input rules to handle user typing: + +```typescript +import { $inputRule } from "@milkdown/kit/utils"; +import { InputRule } from "@milkdown/kit/prose"; + +const markInputRule = $inputRule( + () => + new InputRule(/==(?:{#([^}]+)})?([^=]+)==/, (state, match, start, end) => { + const [okay, color, text] = match; + const { tr } = state; + if (okay) { + tr.addMark( + start, + end, + markSchema.type().create({ color: color || DEFAULT_COLOR }), + ); + } + return tr; + }), +); +``` + +## 4. Color Picker Tooltip + +--- + +To enhance the user experience, we add a color picker tooltip: + +```typescript +export const colorPickerTooltip = tooltipFactory("color-picker"); + +class TooltipPluginView { + // ... implementation +} + +export const colorPickerTooltipConfig = (ctx: Ctx) => { + ctx.set(colorPickerTooltip.key, { + view: () => new TooltipPluginView(ctx), + }); +}; +``` + +## Usage + +--- + +To use the marker plugin, add it to your Milkdown editor configuration: + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; + +Editor.make() + .use(milkdownMarkColorPlugin) + .use(markSchema) + .use(markInputRule) + .use(colorPickerTooltip) + .use(commonmark) + .create(); +``` + +## Example + +--- + +Here's a complete example of the marker plugin in action: + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vanilla-highlight-syntax"} diff --git a/milkdown-docs/plugin/example-slash-plugin.md b/milkdown-docs/plugin/example-slash-plugin.md new file mode 100644 index 0000000..3517d4d --- /dev/null +++ b/milkdown-docs/plugin/example-slash-plugin.md @@ -0,0 +1,137 @@ +# Example: Slash Plugin + +After reading the tooltip guide you already know how Milkdown separates **positioning logic** (provider) from **editor wiring** (ctx slices produced by a factory). +The `@milkdown/plugin-slash` package applies exactly the same idea but focuses on _command palettes_ triggered by a character โ€“ familiar to `/` menus in modern editors. + +This document shows you how to: + +- Understand what the slash plugin gives you out-of-the-box. +- Build a **vanilla TypeScript** implementation of a basic `/` menu. +- Use the slash provider with **React** and **Vue**. +- Explore a full-blown menu feature that ships inside Milkdown's Crepe UI. + +--- + +## 1. Anatomy of a Slash Plugin + +`@milkdown/plugin-slash` exports two utilities: + +1. **`SlashProvider`** โ€“ Measures the caret position and manages show / hide of your menu. +2. **`slashFactory(id)`** โ€“ Generates a ctx slice & ProseMirror plugin pair that plugs the provider into the editor. + +```ts +import { slashFactory } from "@milkdown/plugin-slash"; + +export const [mySlashSpec, mySlashPlugin] = slashFactory("my"); +``` + +Just like the tooltip factory: + +- `mySlashSpec` is where you put a `PluginSpec` (what ProseMirror needs). +- `mySlashPlugin` turns that spec into a runtime plugin. + +--- + +## 2. A Minimal Vanilla `/` Menu + +Below we create a small menu that suggests two commands whenever the user types `/`. + +```ts +import { SlashProvider, slashFactory } from "@milkdown/plugin-slash"; +import { Editor } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; + +// DOM content of the menu โ€“ plain HTML for the demo +const menu = document.createElement("div"); +menu.className = "slash-menu"; +menu.style.cssText = ` + position:absolute;padding:4px 0;background:white;border:1px solid #eee; + box-shadow:0 2px 8px rgba(0,0,0,.15);border-radius:6px;font-size:14px; +`; +menu.innerHTML = `<ul style="margin:0;padding:0;list-style:none"> + <li data-cmd="h1" style="padding:4px 12px;cursor:pointer">Heading 1</li> + <li data-cmd="bullet" style="padding:4px 12px;cursor:pointer">Bullet List</li> +</ul>`; + +// Click handler โ€“ replace with real commands +menu.addEventListener("click", (e) => { + const target = e.target as HTMLElement; + const cmd = target.dataset.cmd; + alert(`Run command: ${cmd}`); +}); + +// Provider positions & shows above DOM element +const provider = new SlashProvider({ + content: menu, + // show the menu when the last character before caret is '/' + shouldShow(view) { + return provider.getContent(view)?.endsWith("/") ?? false; + }, + offset: 8, +}); + +const slash = slashFactory("demo"); +const slashConfig = (ctx: Ctx) => { + ctx.set(slash.key, { + view: () => ({ + update: provider.update, + destroy: provider.destroy, + }), + }); +}; + +Editor.make().config(slashConfig).use(commonmark).use(slash).create(); +``` + +Key takeaways: + +- `SlashProvider` has a helper `getContent(view)` to fetch text before the caret โ€“ handy for filtering. +- You decide **when to show** the menu via the `shouldShow` callback (default: when last char is `/`). +- The provider only manipulates **position + visibility**; rendering & commands are completely yours. + +--- + +## 3. Framework Examples + +### React + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/react-slash"} + +Highlights: + +1. A `<SlashMenu/>` React component renders the list. +2. The component root is passed to `SlashProvider` (just like the tooltip demo). +3. React hooks manage internal focus & keyboard navigation. + +### Vue + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vue-slash"} + +The Vue version uses `Teleport` to append the menu to `document.body` and `ref` / `watch` for reactivity. + +--- + +## 4. Real-world Feature โ€“ Crepe Block Menu + +Milkdown's **Crepe** UI implements an extensible block-level menu on top of the slash plugin. You'll find the source code at: + +```text +packages/crepe/src/feature/block-edit/menu/ +``` + +Notable patterns to look for: + +- **Context slices** (`menu` / `menuAPI`) to expose imperative `show` & `hide` methods. +- Filtering commands based on the current text after `/`. +- Preventing the menu inside `code` blocks or lists. + +Studying this folder is a great next step once you master the basics. + +--- + +## 5. Summary & Next Steps + +- `@milkdown/plugin-slash` gives you caret detection + positioning โ€“ nothing else. +- UI, behaviour, and commands are fully customisable. + +Fork one of the examples above, add your own commands, and you'll have a modern `/` command palette in minutes โœจ. diff --git a/milkdown-docs/plugin/example-tooltip-plugin.md b/milkdown-docs/plugin/example-tooltip-plugin.md new file mode 100644 index 0000000..7b03182 --- /dev/null +++ b/milkdown-docs/plugin/example-tooltip-plugin.md @@ -0,0 +1,140 @@ +# Example: Tooltip Plugin + +This guide walks you through creating and using **tooltip-based plugins** in Milkdown. +You will learn how the low-level `@milkdown/plugin-tooltip` works and how to build richer experiences on top of it in **vanilla TypeScript**, **React**, and **Vue**. + +> **TL;DR** โ€“ A tooltip in Milkdown is nothing more than a ProseMirror plugin created by `tooltipFactory(id)`. +> It receives position information from the editor and renders any DOM of your choice. +> Everything else (buttons, inputs, styling, framework bindings) can be composed on top of that. + +## 1. Anatomy of a Tooltip + +--- + +At its core the tooltip plugin exported from `@milkdown/plugin-tooltip` contains two helpers: + +1. **`TooltipProvider`** โ€“ An utility class powered by [floating-ui](https://floating-ui.com/) to calculate the tooltip position. +2. **`tooltipFactory(id)`** โ€“ A factory that returns a pair of Milkdown plugin slices which wire the provider into the editor. + +The factory is extremely small (โ‰ˆ40 lines): + +```ts +import { tooltipFactory } from "@milkdown/plugin-tooltip"; + +// Create a tooltip identified by the string "my". +export const [myTooltipSpec, myTooltipPlugin] = tooltipFactory("my"); +``` + +The first element (`myTooltipSpec`) is a **ctx slice** that stores a `PluginSpec`, while the second one (`myTooltipPlugin`) is the real ProseMirror plugin which consumes that spec. + +## 2. A Minimal Vanilla Tooltip + +--- + +Below is the complete code for a tooltip that shows the **length of the current selection**. + +```ts +import { Editor } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { TooltipProvider, tooltipFactory } from "@milkdown/plugin-tooltip"; + +// 1) Prepare DOM that we will mount into the page. +const el = document.createElement("div"); +el.className = "selection-length"; +el.style.cssText = ` + pointer-events:none; + background:#333;color:#fff;padding:2px 6px;border-radius:4px;font-size:12px; +`; + +// 2) Build a provider which updates the content. +const provider = new TooltipProvider({ + content: el, + shouldShow: (view) => !!view.state.selection.content().size, +}); + +// 3) Bridge provider & editor. +const tooltip = tooltipFactory("sel-length"); +const tooltipConfig = (ctx: Ctx) => { + ctx.set(selectionTooltipSpec.key, { + view: () => ({ + update: provider.update, + destroy: provider.destroy, + }), + }); +}; + +Editor.make().config(tooltipConfig).use(commonmark).use(tooltip).create(); +``` + +Key points: + +- We **create** any DOM element we like (`el`). +- `TooltipProvider` tracks the editor position and moves the element. +- `tooltipFactory` wraps the provider into a pluggable slice. + +## 3. Framework Examples + +--- + +Sometimes building UI is easier in your favourite framework. +Because the tooltip provider only deals with **DOM elements**, you can freely render React, Vue or Svelte components and pass their root node to the provider. + +### React + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/react-tooltip"} + +The React example shows how to: + +1. Create a React component (`<SelectionTooltip/>`). +2. Render it into a portal and give the root HTML element to `TooltipProvider`. +3. Re-use React state/hooks while Milkdown takes care of positioning. + +### Vue + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vue-tooltip"} + +The Vue example follows the same pattern with `defineComponent` and `teleport`. + +## 4. Real-world Examples + +--- + +### 4-1. Link Tooltip (_@milkdown/component/link-tooltip_) + +The [link tooltip](https://github.com/Milkdown/milkdown/tree/main/packages/components/src/link-tooltip) demonstrates how to: + +- Maintain UI **state** (`preview` vs `edit`) in ctx slices. +- Communicate with the editor through an **API slice** (add / edit / remove links). +- Render framework-agnostic UI inside a tooltip provider. + +Have a look at the files below to see those techniques in action: + +```text +packages/components/src/link-tooltip/ +โ”œโ”€โ”€ slices.ts # state & API slices +โ”œโ”€โ”€ tooltips.ts # preview & edit providers +โ””โ”€โ”€ component.tsx # (framework examples) +``` + +### 4-2. Toolbar Feature (_@milkdown/crepe/feature/toolbar_) + +The toolbar in the [crepe](https://github.com/Milkdown/milkdown/tree/main/packages/crepe) package pushes the idea further by: + +- Using multiple tooltip instances (one per button group). +- Rendering the UI with Vue _inside_ the provider. +- Sharing configuration via ctx slices so that every button is extensible by third-party plugins. + +You can browse the implementation starting from + +```text +packages/crepe/src/feature/toolbar/component.tsx +``` + +## 5. Summary & Next Steps + +--- + +- `@milkdown/plugin-tooltip` offers **just enough** abstraction: positioning & lifecycle. +- Everything else โ€“ **state, styling, framework integration** โ€“ is totally up to you. + +Try to customise one of the examples above, then ship your own tooltip-powered features ๐ŸคŸ. diff --git a/milkdown-docs/plugin/plugins-101.md b/milkdown-docs/plugin/plugins-101.md new file mode 100644 index 0000000..47f72ee --- /dev/null +++ b/milkdown-docs/plugin/plugins-101.md @@ -0,0 +1,174 @@ +# Plugins 101 + +In this section we will show you the basic information of the plugin. +In most cases, you will not need to write plugins without helpers. +But it can help you understand the plugin system and what happens under the hood. + +## Structure Overview + +Generally speaking, a plugin will have following structure: + +```typescript +import { MilkdownPlugin } from "@milkdown/kit/ctx"; + +const myPlugin: MilkdownPlugin = (ctx) => { + // #1 prepare plugin + return async () => { + // #2 run plugin + return async () => { + // #3 clean up plugin + }; + }; +}; +``` + +Each plugin is composed by three parts: + +1. _Prepare_: this part will be executed when plugin is registered in milkdown by `.use` method. +2. _Run_: this part will be executed when plugin is actually loaded. +3. _Post_: this part will be executed when plugin is removed by `.remove` method or editor is destroyed. + +## Timer + +Timer can be used to decide when to load the current plugin and how current plugin can influence other plugin's loading status. + +You can use `ctx.wait` to wait a timer to finish. + +```typescript +import { MilkdownPlugin, Complete } from "@milkdown/kit/core"; + +const myPlugin: MilkdownPlugin = (ctx) => { + return async () => { + const start = Date.now(); + + await ctx.wait(Complete); + + const end = Date.now(); + + console.log("Milkdown load duration: ", end - start); + }; +}; +``` + +You can also create your own timer and influence other plugins load time. +For example, let's create a plugin that will fetch markdown content from remote server as editor's default value. + +```typescript +import { + MilkdownPlugin, + editorStateTimerCtx, + defaultValueCtx, + createTimer, +} from "@milkdown/kit/core"; + +const RemoteTimer = createTimer("RemoteTimer"); + +const remotePlugin: MilkdownPlugin = (ctx) => { + // register timer + ctx.record(RemoteTimer); + + return async () => { + // the editorState plugin will wait for this timer to finish before initialize editor state. + ctx.update(editorStateTimerCtx, (timers) => timers.concat(RemoteTimer)); + + const defaultMarkdown = await fetchMarkdownAPI(); + ctx.set(defaultValueCtx, defaultMarkdown); + + // mark timer as complete + ctx.done(RemoteTimer); + + return async () => { + await SomeAPI(); + + // remove timer when plugin is removed + ctx.clearTimer(RemoteTimer); + }; + }; +}; +``` + +It has following steps: + +1. We use `createTimer` to create a timer, and use `pre.record` to register it into milkdown. +2. We update `editorStateTimerCtx` to tell the internal `editorState` plugin that before initialize editor state, it should wait our remote fetch process finished. +3. After we get value from `fetchMarkdownAPI`, we set it as `defaultValue` and use `ctx.done` to mark a timer as complete. + +## Ctx + +We have used `ctx` several times in the above example, now we can try to understand what it is. + +Ctx is a data container which is shared in the entire editor instance. It's composed by a lot of slices. Every `slice` has a unique key and a value. You can change the value of a slice by `ctx.set` and `ctx.update`. And you can get the value of a slice by `ctx.get` with the slice key or name. Last but not least, you can remove a slice by `post.remove`. + +```typescript +import { MilkdownPlugin, createSlice } from "@milkdown/kit/ctx"; + +const counterCtx = createSlice(0, "counter"); + +const counterPlugin: MilkdownPlugin = (ctx) => { + ctx.inject(counterCtx); + + return () => { + // count is 0 + const count0 = ctx.get(counterCtx); + + // set count to 1 + ctx.set(counterCtx, 1); + + // now count is 1 + const count1 = ctx.get(counterCtx); + + // set count to n + 2 + ctx.update(counterCtx, (prev) => prev + 2); + + // now count is 3 + const count2 = ctx.get(counterCtx); + // we can also get value by the slice name + const count3 = ctx.get("counter"); + + return () => { + // remove the slice + ctx.remove(counterCtx); + }; + }; +}; +``` + +We can use `createSlice` to create a ctx, and use `pre.inject` to inject the ctx into the editor. + +And when plugin processing, `ctx.get` can get the value of a ctx, `ctx.set` can set the value of a ctx, and `ctx.update` can update a ctx using callback function. + +So, we can use `ctx` combine with `timer` to decide when should a plugin be processed. + +```typescript +import { + MilkdownPlugin, + SchemaReady, + Timer, + createSlice, +} from "@milkdown/kit/core"; + +const examplePluginTimersCtx = createSlice<Timer[]>([], "example-timer"); + +const examplePlugin: MilkdownPlugin = (ctx) => { + ctx.inject(examplePluginTimersCtx, [SchemaReady]); + return async () => { + await Promise.all( + ctx.get(examplePluginTimersCtx).map((timer) => ctx.wait(timer)), + ); + // or we can use a simplified syntax sugar + await ctx.waitTimers(examplePluginTimersCtx); + + // do something + }; +}; +``` + +With this pattern, if other plugins want to delay the process of `examplePlugin`, all they need to do is just add a timer into `examplePluginTimersCtx` with `ctx.update`. + +## Summary + +Now let's go back to the plugin structure. Since we have the knowledge of `timer` and `ctx`, we can understand what we should do in each part of a plugin. + +1. In `prepare` stage of the plugin, we can use `ctx.record` to register a timer, and use `ctx.inject` to inject a slice. +2. In `run` stage of the plugin, we can use `ctx.wait` to wait a timer to finish, and use `ctx.get` to get the value of a slice. We can also change values of slices by `ctx.set` and `ctx.update`. And we can use `ctx.done` to mark a timer as complete. +3. In `post` stage of the plugin, we can use `ctx.clearTimer` to clear a timer, and use `ctx.remove` to remove a slice. diff --git a/milkdown-docs/plugin/using-components.md b/milkdown-docs/plugin/using-components.md new file mode 100644 index 0000000..d718cca --- /dev/null +++ b/milkdown-docs/plugin/using-components.md @@ -0,0 +1,28 @@ +# Using Components + +Components are features work out of the box that built on top of plugins. +Each component is a separate module. You can use them by importing them from `@milkdown/kit/component/*`. +All components can be used just like plugins. + +```ts +import { imageBlock } from "@milkdown/kit/component/image-block"; +import { Editor } from "@milkdown/kit/core"; + +Editor.make().use(/* some other plugins */).use(imageBlock).create(); +``` + +Components are designed to be headless, which means they are not opinionated about the UI. +You can use them to build your own editor UI. Components are built by web components and can be used in any framework. + +--- + +# List of Components + +| Name | Description | +| ------------------------------------------------ | ---------------------------------------------------------- | +| [Code Block](/docs/api/component-code-block) | Render code by [Codemirror](https://codemirror.net/) | +| [Image Block](/docs/api/component-image-block) | Render an image as a block | +| [Image Inline](/docs/api/component-image-inline) | Provide placeholder and uploader features for inline image | +| [Link Tooltip](/docs/api/component-link-tooltip) | Provide edit and preview feature for link | +| [List Item](/docs/api/component-list-item-block) | Renderers bullet, ordered and task list by custom renderer | +| [Table Block](/docs/api/component-table-block) | Render table and provides table editing features | diff --git a/milkdown-docs/plugin/using-plugins.md b/milkdown-docs/plugin/using-plugins.md new file mode 100644 index 0000000..cff3cf7 --- /dev/null +++ b/milkdown-docs/plugin/using-plugins.md @@ -0,0 +1,87 @@ +# Using Plugins + +All features in milkdown are provided by plugin. +Such as syntax, components, etc. +Now we can try more plugins: + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { slash } from "@milkdown/kit/plugin/slash"; +import { tooltip } from "@milkdown/kit/plugin/tooltip"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; + +Editor.make().use(commonmark).use(tooltip).use(slash).create(); +``` + +--- + +## Toggling Plugins + +You can also toggle plugins programmatically: + +```typescript +import { Editor } from "@milkdown/kit/core"; +import { someMilkdownPlugin } from "some-milkdown-plugin"; + +const editor = await Editor.config(configForPlugin) + .use(someMilkdownPlugin) + .create(); + +// remove plugin +await editor.remove(someMilkdownPlugin); + +// remove config +editor.removeConfig(configForPlugin); + +// add another plugin +editor.use(anotherMilkdownPlugin); + +// Recreate the editor to apply changes. +await editor.create(); +``` + +--- + +## Official Plugins + +Milkdown provides the following official plugins: + +### Plugins provided by `@milkdown/kit`: + +> ๐Ÿ™‹โ€โ™€๏ธWhy not all plugins are available in `@milkdown/kit`? +> +> `@milkdown/kit` is a collection of plugins that are commonly used in the editor. +> If you want to use a plugin that is not in `@milkdown/kit`, you can install it separately. +> The plugins in `@milkdown/kit` are also stable and well-tested. + +| Package Name | Description | +| -------------------------------------------------------------- | --------------------------------------------------------- | +| [@milkdown/kit/preset/commonmark](/docs/api/preset-commonmark) | Add [commonmark](https://commonmark.org/) syntax support. | +| [@milkdown/kit/preset/gfm](/docs/api/preset-gfm) | Add [gfm](https://github.github.com/gfm/) syntax support. | +| [@milkdown/kit/plugin/history](/docs/api/plugin-history) | Add undo & redo support. | +| [@milkdown/kit/plugin/clipboard](/docs/api/plugin-clipboard) | Add markdown copy & paste support. | +| [@milkdown/kit/plugin/cursor](/docs/api/plugin-cursor) | Add drop & gap cursor. | +| [@milkdown/kit/plugin/listener](/docs/api/plugin-listener) | Add listener support. | +| [@milkdown/kit/plugin/indent](/docs/api/plugin-indent) | Add tab indent support. | +| [@milkdown/kit/plugin/upload](/docs/api/plugin-upload) | Add drop and upload support. | +| [@milkdown/kit/plugin/block](/docs/api/plugin-block) | Add a drag handle for every block node. | +| [@milkdown/kit/plugin/tooltip](/docs/api/plugin-tooltip) | Add universal tooltip support. | +| [@milkdown/kit/plugin/slash](/docs/api/plugin-slash) | Add universal slash commands support. | + +### Other Plugins: + +- [@milkdown/plugin-collab](/docs/api/plugin-collab) + + Add collaborative editing support, powered by [yjs](https://docs.yjs.dev/). + +- [@milkdown/plugin-prism](/docs/api/plugin-prism) + + Add [prism](https://prismjs.com/) support for code block highlight. + +- [@milkdown/plugin-emoji](/docs/api/plugin-emoji) + + Add emoji shortcut support (something like `:+1:`), and use [twemoji](https://twemoji.twitter.com/) to display emoji. + +## Community plugins + +Check out [awesome-milkdown](https://github.com/Milkdown/awesome-milkdown) to find community plugins. You can also submit a PR to list your plugins there. diff --git a/milkdown-docs/recipes/angular.md b/milkdown-docs/recipes/angular.md new file mode 100644 index 0000000..8824e00 --- /dev/null +++ b/milkdown-docs/recipes/angular.md @@ -0,0 +1,48 @@ +# Angular + +We don't provide Angular support out of box, but you can use the vanilla version with it easily. + +## Install the Dependencies + +```bash +# install with npm +npm install @milkdown/kit +npm install @milkdown/theme-nord +``` + +## Create a Component + +Create a component is pretty easy. + +```html +<!-- editor.component.html --> +<div #editorRef></div> +``` + +```typescript +// editor.component.ts +import { Component, ElementRef, ViewChild } from "@angular/core"; +import { defaultValueCtx, Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { nord } from "@milkdown/theme-nord"; + +@Component({ + templateUrl: "./editor.component.html", +}) +export class AppComponent { + @ViewChild("editorRef") editorRef: ElementRef; + + defaultValue = "# Milkdown x Angular"; + + ngAfterViewInit() { + Editor.make() + .config((ctx) => { + ctx.set(rootCtx, this.editorRef.nativeElement); + ctx.set(defaultValueCtx, this.defaultValue); + }) + .config(nord) + .use(commonmark) + .create(); + } +} +``` diff --git a/milkdown-docs/recipes/nextjs.md b/milkdown-docs/recipes/nextjs.md new file mode 100644 index 0000000..2d616f0 --- /dev/null +++ b/milkdown-docs/recipes/nextjs.md @@ -0,0 +1,51 @@ +# Next.js + +Since we provide [react](/docs/recipes/react) support out of box, we can use it directly in [Next.js](https://nextjs.org/). + +## Install the Dependencies + +Except the `@milkdown/kit` and theme. We need to install the `@milkdown/react`, which provide lots of abilities for react in milkdown. + +```bash +# install with npm +npm install @milkdown/react +npm install @milkdown/kit +npm install @milkdown/theme-nord +``` + +## Create a Component + +Create a component is pretty easy. + +```tsx +import { Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { Milkdown, MilkdownProvider, useEditor } from "@milkdown/react"; +import { nord } from "@milkdown/theme-nord"; +import React from "react"; + +const MilkdownEditor: React.FC = () => { + const { editor } = useEditor((root) => + Editor.make() + .config(nord) + .config((ctx) => { + ctx.set(rootCtx, root); + }) + .use(commonmark), + ); + + return <Milkdown />; +}; + +export const MilkdownEditorWrapper: React.FC = () => { + return ( + <MilkdownProvider> + <MilkdownEditor /> + </MilkdownProvider> + ); +}; +``` + +## Online Demo + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/next-commonmark"} diff --git a/milkdown-docs/recipes/nuxtjs.md b/milkdown-docs/recipes/nuxtjs.md new file mode 100644 index 0000000..391da10 --- /dev/null +++ b/milkdown-docs/recipes/nuxtjs.md @@ -0,0 +1,82 @@ +# NuxtJS + +Since we provide [vue](/docs/recipes/vue) support out of box, we can use it directly in [NuxtJS](https://v3.nuxtjs.org/). + +> NuxtJS version should be 3.x. + +## Install the Dependencies + +Except the `@milkdown/kit` and theme. We need to install the `@milkdown/vue`, which provide lots of abilities for vue in milkdown. + +```bash +# install with npm +npm install @milkdown/vue +npm install @milkdown/kit +npm install @milkdown/theme-nord +``` + +## Create a Component + +Create a component is pretty easy. + +First, we need to create a `MilkdownEditor` component. + +```html +<!-- MilkdownEditor.vue --> +<template> + <Milkdown /> +</template> + +<script> + import { Editor, rootCtx, defaultValueCtx } from "@milkdown/kit/core"; + import { commonmark } from "@milkdown/kit/preset/commonmark"; + import { nord } from "@milkdown/theme-nord"; + import { Milkdown, useEditor } from "@milkdown/vue"; + import { defineComponent } from "vue"; + + export default defineComponent({ + name: "Milkdown", + components: { + Milkdown, + }, + setup: () => { + useEditor((root) => + Editor.make() + .config((ctx) => { + ctx.set(rootCtx, root); + }) + .config(nord) + .use(commonmark), + ); + }, + }); +</script> +``` + +Then, we need to create a `MilkdownEditorWrapper` component. + +```html +<!-- MilkdownEditorWrapper.vue --> +<template> + <MilkdownProvider> + <MilkdownEditor /> + </MilkdownProvider> +</template> + +<script> + import { MilkdownProvider } from "@milkdown/vue"; + import { defineComponent } from "vue"; + + export default defineComponent({ + name: "MilkdownEditorWrapper", + components: { + MilkdownProvider, + }, + setup: () => {}, + }); +</script> +``` + +## Online Demo + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/nuxt-commonmark"} diff --git a/milkdown-docs/recipes/react.md b/milkdown-docs/recipes/react.md new file mode 100644 index 0000000..45e1683 --- /dev/null +++ b/milkdown-docs/recipes/react.md @@ -0,0 +1,213 @@ +# React Integration + +Milkdown provides first-class React support with dedicated packages and hooks for seamless integration. You can choose between Crepe, our feature-rich WYSIWYG editor, or the core Milkdown editor for more customization options. + +## Using Crepe + +--- + +Crepe is a powerful, feature-rich Markdown editor built on top of Milkdown that provides a more user-friendly editing experience. + +### Installation + +```bash +npm install @milkdown/crepe @milkdown/react @milkdown/kit +``` + +### Implementation + +```tsx +import { Crepe } from "@milkdown/crepe"; +import { Milkdown, MilkdownProvider, useEditor } from "@milkdown/react"; + +const CrepeEditor: React.FC = () => { + const { get } = useEditor((root) => { + return new Crepe({ root }); + }); + + return <Milkdown />; +}; + +export const MilkdownEditorWrapper: React.FC = () => { + return ( + <MilkdownProvider> + <CrepeEditor /> + </MilkdownProvider> + ); +}; +``` + +### Online Demo + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/react-crepe"} + +## Using Milkdown + +--- + +For more advanced use cases or when you need full control over the editor's configuration, you can use the core Milkdown editor directly. + +### Install Dependencies + +```bash +npm install @milkdown/react @milkdown/kit +``` + +### Basic Usage + +Here's a minimal example to get started: + +```tsx +import { Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { Milkdown, MilkdownProvider, useEditor } from "@milkdown/react"; +import { nord } from "@milkdown/theme-nord"; + +const MilkdownEditor: React.FC = () => { + const { get } = useEditor((root) => + Editor.make() + .config(nord) + .config((ctx) => { + ctx.set(rootCtx, root); + }) + .use(commonmark), + ); + + return <Milkdown />; +}; + +export const MilkdownEditorWrapper: React.FC = () => { + return ( + <MilkdownProvider> + <MilkdownEditor /> + </MilkdownProvider> + ); +}; +``` + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/react-commonmark"} + +## Advanced Usage + +--- + +### Accessing Editor Instance + +The `useInstance()` hook can only be used within components that are children of `MilkdownProvider`. It returns a tuple containing a loading state and a getter function to access the editor instance. + +```tsx +import { useInstance } from "@milkdown/react"; +import { getMarkdown } from "@milkdown/utils"; + +// โŒ This won't work - ParentComponent is outside MilkdownProvider +const ParentComponent: React.FC = () => { + const [isLoading, getInstance] = useInstance(); // This will be [true, () => undefined] + return <MilkdownEditorWrapper />; +}; + +// โœ… This is the correct way - EditorControls is inside MilkdownProvider +const EditorControls: React.FC = () => { + const [isLoading, getInstance] = useInstance(); + + const handleSave = () => { + if (isLoading) return; + + const editor = getInstance(); + if (!editor) return; + + const content = editor.action(getMarkdown()); + // Do something with the content + }; + + return ( + <button onClick={handleSave} disabled={isLoading}> + Save + </button> + ); +}; + +// โœ… Proper component structure +const EditorWithControls: React.FC = () => { + return ( + <MilkdownProvider> + <MilkdownEditorWrapper /> + <EditorControls /> + </MilkdownProvider> + ); +}; +``` + +### Best Practices + +1. **Component Structure** + - Keep the editor component separate from business logic + - Wrap the editor with `MilkdownProvider` at the highest necessary level + - Use TypeScript for better type safety + +2. **Performance** + - Memoize the editor configuration if it's complex + - Use React.memo for the editor component if needed + - Avoid unnecessary re-renders of the editor + +### Common Use Cases + +**Form Integration** + +```tsx +const FormWithEditor: React.FC = () => { + const [isLoading, getInstance] = useInstance(); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (isLoading) return; + + const editor = getInstance(); + if (!editor) return; + + const content = editor.action(getMarkdown()); + // Submit form with content + }; + + return ( + <form onSubmit={handleSubmit}> + <MilkdownEditorWrapper /> + <button type="submit" disabled={isLoading}> + Submit + </button> + </form> + ); +}; +``` + +**Auto-save** + +```tsx +import { Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { listener, listenerCtx } from "@milkdown/kit/plugin/listener"; +import { Milkdown, useEditor } from "@milkdown/react"; + +const AutoSaveEditor: React.FC = () => { + const { get } = useEditor((root) => + Editor.make() + .config((ctx) => { + ctx.set(rootCtx, root); + // Add markdown listener for auto-save + ctx.get(listenerCtx).markdownUpdated((ctx, markdown) => { + // Save content to your backend or storage + saveToBackend(markdown); + }); + }) + .use(commonmark) + .use(listener), + ); + + return <Milkdown />; +}; +``` + +## More Examples + +--- + +- [Examples Repository](https://github.com/Milkdown/examples) diff --git a/milkdown-docs/recipes/solidjs.md b/milkdown-docs/recipes/solidjs.md new file mode 100644 index 0000000..a806767 --- /dev/null +++ b/milkdown-docs/recipes/solidjs.md @@ -0,0 +1,46 @@ +# SolidJS + +We don't provide SolidJS support out of box, but you can use the vanilla version with it easily. + +## Install the Dependencies + +```bash +# install with npm +npm install @milkdown/kit +npm install @milkdown/theme-nord +``` + +## Create a Component + +Create a component is pretty easy. + +```tsx +import { defaultValueCtx, Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { nord } from "@milkdown/theme-nord"; +import { onCleanup, onMount } from "solid-js"; + +const Milkdown = () => { + let ref; + let editor; + onMount(async () => { + editor = await Editor.make() + .config((ctx) => { + ctx.set(rootCtx, ref); + }) + .config(nord) + .use(commonmark) + .create(); + }); + + onCleanup(() => { + editor.destroy(); + }); + + return <div ref={ref} />; +}; +``` + +## Online Demo + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/solid-commonmark"} diff --git a/milkdown-docs/recipes/svelte.md b/milkdown-docs/recipes/svelte.md new file mode 100644 index 0000000..079bb7d --- /dev/null +++ b/milkdown-docs/recipes/svelte.md @@ -0,0 +1,45 @@ +# Svelte + +We don't provide Svelte support out of box, but you can use the vanilla version with it easily. + +## Install the Dependencies + +```bash +# install with npm +npm install @milkdown/kit +npm install @milkdown/theme-nord +``` + +## Creating a Component + +Creating a component is pretty easy. + +```html +<script> + import { Editor, rootCtx, defaultValueCtx } from "@milkdown/kit/core"; + import { commonmark } from "@milkdown/kit/preset/commonmark"; + import { nord } from "@milkdown/theme-nord"; + + function editor(dom) { + // to obtain the editor instance we need to store a reference of the editor. + const MakeEditor = Editor.make() + .config((ctx) => { + ctx.set(rootCtx, dom); + }) + .config(nord) + .use(commonmark) + .create(); + MakeEditor.then((editor) => { + // here you have access to the editor instance. + // const exampleContent = "# Hello World!"; + // editor.action(replaceAll(exampleContent)); + }); + } +</script> + +<div use:editor /> +``` + +## Online Demo + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/svelte-commonmark"} diff --git a/milkdown-docs/recipes/vue.md b/milkdown-docs/recipes/vue.md new file mode 100644 index 0000000..de05ca0 --- /dev/null +++ b/milkdown-docs/recipes/vue.md @@ -0,0 +1,294 @@ +# Vue Integration + +Milkdown provides first-class Vue support with dedicated packages and hooks for seamless integration. You can choose between Crepe, our feature-rich WYSIWYG editor, or the core Milkdown editor for more customization options. + +> Vue version should be 3.x + +## Using Crepe + +--- + +Crepe is a powerful, feature-rich Markdown editor built on top of Milkdown that provides a more user-friendly editing experience. + +### Installation + +```bash +npm install @milkdown/crepe @milkdown/vue @milkdown/kit +``` + +### Implementation + +```vue +<!-- MilkdownEditor.vue --> +<template> + <Milkdown /> +</template> + +<script> +import { Crepe } from "@milkdown/crepe"; +import { Milkdown, MilkdownProvider, useEditor } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "MilkdownEditor", + components: { + Milkdown, + }, + setup: () => { + const { get } = useEditor((root) => { + return new Crepe({ root }); + }); + }, +}); +</script> + +<!-- MilkdownEditorWrapper.vue --> +<template> + <MilkdownProvider> + <MilkdownEditor /> + </MilkdownProvider> +</template> + +<script> +import { MilkdownProvider } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "MilkdownEditorWrapper", + components: { + MilkdownProvider, + }, +}); +</script> +``` + +### Online Demo + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vue-crepe"} + +## Using Milkdown + +--- + +For more advanced use cases or when you need full control over the editor's configuration, you can use the core Milkdown editor directly. + +### Install Dependencies + +```bash +npm install @milkdown/vue @milkdown/kit @milkdown/theme-nord +``` + +### Basic Usage + +Here's a minimal example to get started: + +```vue +<!-- MilkdownEditor.vue --> +<template> + <Milkdown /> +</template> + +<script> +import { Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { nord } from "@milkdown/theme-nord"; +import { Milkdown, useEditor } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "MilkdownEditor", + components: { + Milkdown, + }, + setup: () => { + const { get } = useEditor((root) => + Editor.make() + .config(nord) + .config((ctx) => { + ctx.set(rootCtx, root); + }) + .use(commonmark), + ); + }, +}); +</script> + +<!-- MilkdownEditorWrapper.vue --> +<template> + <MilkdownProvider> + <MilkdownEditor /> + </MilkdownProvider> +</template> + +<script> +import { MilkdownProvider } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "MilkdownEditorWrapper", + components: { + MilkdownProvider, + }, +}); +</script> +``` + +::iframe{src="https://stackblitz.com/github/Milkdown/examples/tree/main/vue-commonmark"} + +## Advanced Usage + +--- + +### Accessing Editor Instance + +The `useInstance()` hook can only be used within components that are children of `MilkdownProvider`. It returns a tuple containing a loading state and a getter function to access the editor instance. + +```vue +<!-- EditorControls.vue --> +<template> + <button @click="handleSave" :disabled="isLoading">Save</button> +</template> + +<script> +import { useInstance } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "EditorControls", + setup: () => { + const [isLoading, getInstance] = useInstance(); + + const handleSave = () => { + if (isLoading.value) return; + + const editor = getInstance(); + if (!editor) return; + + const content = editor.getMarkdown(); + // Do something with the content + }; + + return { + isLoading, + handleSave, + }; + }, +}); +</script> + +<!-- EditorWithControls.vue --> +<template> + <MilkdownProvider> + <MilkdownEditor /> + <EditorControls /> + </MilkdownProvider> +</template> + +<script> +import { MilkdownProvider } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "EditorWithControls", + components: { + MilkdownProvider, + }, +}); +</script> +``` + +### Best Practices + +1. **Component Structure** + - Keep the editor component separate from business logic + - Wrap the editor with `MilkdownProvider` at the highest necessary level + - Use TypeScript for better type safety + +2. **Performance** + - Memoize the editor configuration if it's complex + - Use Vue's `shallowRef` for editor instance if needed + - Avoid unnecessary re-renders of the editor + +### Common Use Cases + +**Form Integration** + +```vue +<template> + <form @submit.prevent="handleSubmit"> + <MilkdownEditorWrapper /> + <button type="submit" :disabled="isLoading">Submit</button> + </form> +</template> + +<script> +import { useInstance } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "FormWithEditor", + setup: () => { + const [isLoading, getInstance] = useInstance(); + + const handleSubmit = () => { + if (isLoading.value) return; + + const editor = getInstance(); + if (!editor) return; + + const content = editor.getMarkdown(); + // Submit form with content + }; + + return { + isLoading, + handleSubmit, + }; + }, +}); +</script> +``` + +**Auto-save** + +```vue +<template> + <Milkdown /> +</template> + +<script> +import { Editor, rootCtx } from "@milkdown/kit/core"; +import { commonmark } from "@milkdown/kit/preset/commonmark"; +import { listener, listenerCtx } from "@milkdown/kit/plugin/listener"; +import { Milkdown, useEditor } from "@milkdown/vue"; +import { defineComponent } from "vue"; + +export default defineComponent({ + name: "AutoSaveEditor", + components: { + Milkdown, + }, + setup: () => { + const { get } = useEditor((root) => + Editor.make() + .config((ctx) => { + ctx.set(rootCtx, root); + // Add markdown listener for auto-save + ctx.get(listenerCtx).markdownUpdated((ctx, markdown) => { + // Save content to your backend or storage + saveToBackend(markdown); + }); + }) + .use(commonmark) + .use(listener), + ); + }, +}); +</script> +``` + +## More Examples + +--- + +- [Examples Repository](https://github.com/Milkdown/examples) diff --git a/milkdown-docs/recipes/vue2.md b/milkdown-docs/recipes/vue2.md new file mode 100644 index 0000000..71575f0 --- /dev/null +++ b/milkdown-docs/recipes/vue2.md @@ -0,0 +1,44 @@ +# Vue2 + +We don't provide Vue2 support out of box, but you can use the vanilla version with it easily. + +## Install the Dependencies + +```bash +# install with npm +npm install @milkdown/kit +npm install @milkdown/theme-nord +``` + +## Create a Component + +Create a component is pretty easy. + +```html +<template> + <div ref="editor"></div> +</template> + +<script> + import { defaultValueCtx, Editor, rootCtx } from "@milkdown/kit/core"; + import { commonmark } from "@milkdown/kit/preset/commonmark"; + import { nord } from "@milkdown/theme-nord"; + + export default { + name: "Editor", + props: { + msg: String, + }, + mounted() { + Editor.make() + .config((ctx) => { + ctx.set(rootCtx, this.$refs.editor); + ctx.set(defaultValueCtx, this.$props.msg); + }) + .config(nord) + .use(commonmark) + .create(); + }, + }; +</script> +```