Add tokenization and context provider API types

- Implemented window delineation tests for indentation-based tokenization.
- Created tokenizer module with various tokenization strategies including TTokenizer and ApproximateTokenizer.
- Added type definitions for authentication parameters and code citation notifications.
- Introduced context provider API for extensions to supply additional context items to Copilot.
- Defined core types and schemas for position and range.
- Established status types for agent status management in IDEs.

completions-sample-code/prompt/src/test/snippets.test.ts

@@ -0,0 +1,32 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import { SnippetProviderType, SnippetSemantics, announceSnippet } from '../snippetInclusion/snippets';
+import * as assert from 'assert';
+import dedent from 'ts-dedent';
+
+suite('Unit tests for snippet.ts', () => {
+	const bogusSnippet = {
+		relativePath: 'snippet1.ts',
+		score: 1.0,
+		startLine: 1,
+		endLine: 3,
+		provider: SnippetProviderType.Path,
+		semantics: SnippetSemantics.Snippet,
+		snippet: dedent`
+				A
+					B
+					C`,
+	};
+
+	test('announceSnippet', function () {
+		assert.deepStrictEqual(announceSnippet(bogusSnippet), {
+			headline: 'Compare this snippet from snippet1.ts:',
+			snippet: dedent`
+				A
+					B
+					C`,
+		});
+	});
+});