Add tokenization and context provider API types

- Implemented window delineation tests for indentation-based tokenization.
- Created tokenizer module with various tokenization strategies including TTokenizer and ApproximateTokenizer.
- Added type definitions for authentication parameters and code citation notifications.
- Introduced context provider API for extensions to supply additional context items to Copilot.
- Defined core types and schemas for position and range.
- Established status types for agent status management in IDEs.

completions-sample-code/extension/src/extensionStatus.ts

@@ -0,0 +1,27 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { createServiceIdentifier } from '../../../../../util/common/services';
+import { Command, StatusKind } from '../../types/src';
+
+export const ICompletionsExtensionStatus = createServiceIdentifier<ICompletionsExtensionStatus>('ICompletionsExtensionStatus');
+export interface ICompletionsExtensionStatus {
+	readonly _serviceBrand: undefined;
+
+	kind: StatusKind;
+	message?: string;
+	busy: boolean;
+	command?: Command;
+}
+
+export class CopilotExtensionStatus implements ICompletionsExtensionStatus {
+	declare _serviceBrand: undefined;
+	constructor(
+		public kind: StatusKind = 'Normal',
+		public message?: string,
+		public busy = false,
+		public command?: Command
+	) { }
+}