Skip to content

Commit f29d24a

Browse files
icycodesicycodes
authored
feat(vscode): support collecting relevant snippets from recenlty changed file. (#1844)
* feat(vscode): support collecting relevant snippets from recenlty changed file. * fix: update code search strategy.
1 parent 97454fa commit f29d24a

File tree

14 files changed

+617
-43
lines changed

14 files changed

+617
-43
lines changed

‎clients/tabby-agent/openapi/tabby.json‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"info": {
44
"title": "Tabby Server",
55
"description": "\n[![tabby stars](https://img.shields.io/github/stars/TabbyML/tabby)](https://github.com/TabbyML/tabby)\n[![Join Slack](https://shields.io/badge/Join-Tabby%20Slack-red?logo=slack)](https://links.tabbyml.com/join-slack)\n\nInstall following IDE / Editor extensions to get started with [Tabby](https://github.com/TabbyML/tabby).\n* [VSCode Extension](https://github.com/TabbyML/tabby/tree/main/clients/vscode) – Install from the [marketplace](https://marketplace.visualstudio.com/items?itemName=TabbyML.vscode-tabby), or [open-vsx.org](https://open-vsx.org/extension/TabbyML/vscode-tabby)\n* [VIM Extension](https://github.com/TabbyML/tabby/tree/main/clients/vim)\n* [IntelliJ Platform Plugin](https://github.com/TabbyML/tabby/tree/main/clients/intellij) – Install from the [marketplace](https://plugins.jetbrains.com/plugin/22379-tabby)\n",
6+
"contact": { "name": "TabbyML Team" },
67
"license": { "name": "Apache 2.0", "url": "https://github.com/TabbyML/tabby/blob/main/LICENSE" },
78
"version": "0.10.0-dev.0"
89
},
@@ -80,13 +81,13 @@
8081
"name": "limit",
8182
"in": "query",
8283
"required": false,
83-
"schema": { "type": "integer", "default": 20, "nullable": true, "minimum": 0.0 }
84+
"schema": { "type": "integer", "default": 20, "nullable": true, "minimum": 0 }
8485
},
8586
{
8687
"name": "offset",
8788
"in": "query",
8889
"required": false,
89-
"schema": { "type": "integer", "default": 0, "nullable": true, "minimum": 0.0 }
90+
"schema": { "type": "integer", "default": 0, "nullable": true, "minimum": 0 }
9091
}
9192
],
9293
"responses": {
@@ -119,7 +120,7 @@
119120
"type": "object",
120121
"required": ["index", "delta"],
121122
"properties": {
122-
"index": { "type": "integer", "minimum": 0.0 },
123+
"index": { "type": "integer", "minimum": 0 },
123124
"logprobs": { "type": "string", "nullable": true },
124125
"finish_reason": { "type": "string", "nullable": true },
125126
"delta": { "$ref": "#/components/schemas/ChatCompletionDelta" }
@@ -130,7 +131,7 @@
130131
"required": ["id", "created", "system_fingerprint", "object", "model", "choices"],
131132
"properties": {
132133
"id": { "type": "string" },
133-
"created": { "type": "integer", "format": "int64", "minimum": 0.0 },
134+
"created": { "type": "integer", "format": "int64", "minimum": 0 },
134135
"system_fingerprint": { "type": "string" },
135136
"object": { "type": "string" },
136137
"model": { "type": "string" },
@@ -148,7 +149,7 @@
148149
"properties": {
149150
"messages": { "type": "array", "items": { "$ref": "#/components/schemas/Message" } },
150151
"temperature": { "type": "number", "format": "float", "nullable": true },
151-
"seed": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0.0 }
152+
"seed": { "type": "integer", "format": "int64", "nullable": true, "minimum": 0 }
152153
},
153154
"example": {
154155
"messages": [
@@ -161,10 +162,7 @@
161162
"Choice": {
162163
"type": "object",
163164
"required": ["index", "text"],
164-
"properties": {
165-
"index": { "type": "integer", "format": "int32", "minimum": 0.0 },
166-
"text": { "type": "string" }
167-
}
165+
"properties": { "index": { "type": "integer", "format": "int32", "minimum": 0 }, "text": { "type": "string" } }
168166
},
169167
"CompletionRequest": {
170168
"type": "object",
@@ -193,7 +191,7 @@
193191
"format": "int64",
194192
"description": "The seed used for randomly selecting tokens",
195193
"nullable": true,
196-
"minimum": 0.0
194+
"minimum": 0
197195
}
198196
},
199197
"example": {
@@ -241,7 +239,7 @@
241239
"properties": {
242240
"filepath": {
243241
"type": "string",
244-
"description": "Filepath of the file where the snippet is from.\n- When the file belongs to the same workspace as the current file,\nthis is a relative filepath, that has the same root as the current file.\n- When the file located outside the workspace, such as in a dependency package,\nthis is a file URI with an absolute filepath."
242+
"description": "Filepath of the file where the snippet is from.\n- When the file belongs to the same workspace as the current file,\nthis is a relative filepath, use the same rule as [Segments::filepath].\n- When the file located outside the workspace, such as in a dependency package,\nthis is a file URI with an absolute filepath."
245243
},
246244
"body": { "type": "string", "description": "Body of the snippet." }
247245
}
@@ -255,7 +253,7 @@
255253
"device": { "type": "string" },
256254
"arch": { "type": "string" },
257255
"cpu_info": { "type": "string" },
258-
"cpu_count": { "type": "integer", "minimum": 0.0 },
256+
"cpu_count": { "type": "integer", "minimum": 0 },
259257
"cuda_devices": { "type": "array", "items": { "type": "string" } },
260258
"version": { "$ref": "#/components/schemas/Version" }
261259
}
@@ -266,7 +264,7 @@
266264
"properties": {
267265
"score": { "type": "number", "format": "float" },
268266
"doc": { "$ref": "#/components/schemas/HitDocument" },
269-
"id": { "type": "integer", "format": "int32", "minimum": 0.0 }
267+
"id": { "type": "integer", "format": "int32", "minimum": 0 }
270268
}
271269
},
272270
"HitDocument": {
@@ -291,9 +289,9 @@
291289
"example": "view"
292290
},
293291
"completion_id": { "type": "string" },
294-
"choice_index": { "type": "integer", "format": "int32", "minimum": 0.0 },
292+
"choice_index": { "type": "integer", "format": "int32", "minimum": 0 },
295293
"view_id": { "type": "string", "nullable": true },
296-
"elapsed": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0.0 }
294+
"elapsed": { "type": "integer", "format": "int32", "nullable": true, "minimum": 0 }
297295
}
298296
},
299297
"Message": {
@@ -305,7 +303,7 @@
305303
"type": "object",
306304
"required": ["num_hits", "hits"],
307305
"properties": {
308-
"num_hits": { "type": "integer", "minimum": 0.0 },
306+
"num_hits": { "type": "integer", "minimum": 0 },
309307
"hits": { "type": "array", "items": { "$ref": "#/components/schemas/Hit" } }
310308
}
311309
},
@@ -321,7 +319,7 @@
321319
},
322320
"filepath": {
323321
"type": "string",
324-
"description": "The relative path of the file that is being edited.\n- When `git_url` is set, this is the path of the file in the git repository.\n- When `git_url` is empty, this is the path of the file in the workspace.",
322+
"description": "The relative path of the file that is being edited.\n- When [Segments::git_url] is set, this is the path of the file in the git repository.\n- When [Segments::git_url] is empty, this is the path of the file in the workspace.",
325323
"nullable": true
326324
},
327325
"git_url": {
@@ -332,7 +330,13 @@
332330
"declarations": {
333331
"type": "array",
334332
"items": { "$ref": "#/components/schemas/Declaration" },
335-
"description": "The relevant declaration code snippets provided by editor.\nIt'll contains declarations extracted from `prefix` segments using LSP.",
333+
"description": "The relevant declaration code snippets provided by the editor's LSP,\ncontain declarations of symbols extracted from [Segments::prefix].",
334+
"nullable": true
335+
},
336+
"relevant_snippets_from_changed_files": {
337+
"type": "array",
338+
"items": { "$ref": "#/components/schemas/Snippet" },
339+
"description": "The relevant code snippets extracted from recently edited files.\nThese snippets are selected from candidates found within code chunks\nbased on the edited location.\nThe current editing file is excluded from the search candidates.\n\nWhen provided alongside [Segments::declarations], the snippets have\nalready been deduplicated to ensure no duplication with entries\nin [Segments::declarations].\n\nSorted in descending order of [Snippet::score].",
336340
"nullable": true
337341
},
338342
"clipboard": {

‎clients/tabby-agent/src/AgentConfig.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,30 @@ export type AgentConfig = {
1717
// max number of characters per snippet
1818
maxCharsPerSnippet: number;
1919
};
20+
collectSnippetsFromRecentChangedFiles: {
21+
enabled: boolean;
22+
// max number of snippets
23+
maxSnippets: number;
24+
indexing: {
25+
// Interval in ms for indexing worker to check pending task
26+
checkingChangesInterval: number;
27+
// Debouncing interval in ms for sending changes to indexing task
28+
changesDebouncingInterval: number;
29+
30+
// Determine the crop window at changed location for indexing
31+
// Line before changed location
32+
prefixLines: number;
33+
// Line after changed location
34+
suffixLines: number;
35+
36+
// Max number of chunks in memory
37+
maxChunks: number;
38+
// chars per code chunk
39+
chunkSize: number;
40+
// overlap lines between neighbor chunks
41+
overlapLines: number;
42+
};
43+
};
2044
clipboard: {
2145
minChars: number;
2246
maxChars: number;
@@ -82,6 +106,19 @@ export const defaultAgentConfig: AgentConfig = {
82106
maxSnippets: 5,
83107
maxCharsPerSnippet: 500,
84108
},
109+
collectSnippetsFromRecentChangedFiles: {
110+
enabled: false,
111+
maxSnippets: 3,
112+
indexing: {
113+
checkingChangesInterval: 500,
114+
changesDebouncingInterval: 1000,
115+
prefixLines: 20,
116+
suffixLines: 20,
117+
maxChunks: 100,
118+
chunkSize: 500,
119+
overlapLines: 1,
120+
},
121+
},
85122
clipboard: {
86123
minChars: 3,
87124
maxChars: 2000,

‎clients/tabby-agent/src/CompletionContext.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,22 @@ export type CompletionRequest = {
1818
}[];
1919
};
2020
declarations?: Declaration[];
21+
relevantSnippetsFromChangedFiles?: CodeSnippet[];
2122
};
2223

2324
export type Declaration = {
2425
filepath: string;
26+
offset: number;
2527
text: string;
2628
};
2729

30+
export type CodeSnippet = {
31+
filepath: string;
32+
offset: number;
33+
text: string;
34+
score: number;
35+
};
36+
2837
export type CompletionResponseChoice = {
2938
index: number;
3039
text: string;
@@ -71,6 +80,7 @@ export class CompletionContext {
7180
};
7281

7382
declarations?: Declaration[];
83+
relevantSnippetsFromChangedFiles?: CodeSnippet[];
7484

7585
// "default": the cursor is at the end of the line
7686
// "fill-in-line": the cursor is not at the end of the line, except auto closed characters
@@ -98,6 +108,7 @@ export class CompletionContext {
98108
this.git = request.git;
99109

100110
this.declarations = request.declarations;
111+
this.relevantSnippetsFromChangedFiles = request.relevantSnippetsFromChangedFiles;
101112

102113
const lineEnd = isAtLineEndExcludingAutoClosedChar(this.suffixLines[0] ?? "");
103114
this.mode = lineEnd ? "default" : "fill-in-line";
@@ -108,6 +119,7 @@ export class CompletionContext {
108119
position: this.position,
109120
clipboard: this.clipboard,
110121
declarations: this.declarations,
122+
relevantSnippetsFromChangedFiles: this.relevantSnippetsFromChangedFiles,
111123
});
112124
}
113125
}

‎clients/tabby-agent/src/TabbyAgent.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,34 @@ export class TabbyAgent extends EventEmitter implements Agent {
374374
};
375375
});
376376

377+
// snippets
378+
const relevantSnippetsFromChangedFiles = context.relevantSnippetsFromChangedFiles
379+
// deduplicate
380+
?.filter(
381+
(snippet) =>
382+
// Remove snippet if find a declaration from the same file and range is overlapping
383+
!context.declarations?.find((declaration) => {
384+
return (
385+
declaration.filepath === snippet.filepath &&
386+
// Is range overlapping
387+
Math.max(declaration.offset, snippet.offset) <=
388+
Math.min(declaration.offset + declaration.text.length, snippet.offset + snippet.text.length)
389+
);
390+
}),
391+
)
392+
.map((snippet) => {
393+
let snippetFilepath = snippet.filepath;
394+
if (relativeFilepathRoot && snippetFilepath.startsWith(relativeFilepathRoot)) {
395+
snippetFilepath = path.relative(relativeFilepathRoot, snippetFilepath);
396+
}
397+
return {
398+
filepath: snippetFilepath,
399+
body: snippet.text,
400+
score: snippet.score,
401+
};
402+
})
403+
.sort((a, b) => b.score - a.score);
404+
377405
// clipboard
378406
let clipboard = undefined;
379407
const clipboardConfig = this.config.completion.prompt.clipboard;
@@ -386,6 +414,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
386414
filepath,
387415
git_url: gitUrl,
388416
declarations,
417+
relevant_snippets_from_changed_files: relevantSnippetsFromChangedFiles,
389418
clipboard,
390419
};
391420
}

‎clients/tabby-agent/src/configFile.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,21 @@ const typeCheckSchema: Record<string, string> = {
5353
"completion.prompt.experimentalStripAutoClosingCharacters": "boolean",
5454
"completion.prompt.maxPrefixLines": "number",
5555
"completion.prompt.maxSuffixLines": "number",
56-
"completion.prompt.experimentalDeclarations": "object",
57-
"completion.prompt.experimentalDeclarations.enabled": "boolean",
58-
"completion.prompt.experimentalDeclarations.maxSnippets": "number",
59-
"completion.prompt.experimentalDeclarations.maxChars": "number",
56+
"completion.prompt.fillDeclarations": "object",
57+
"completion.prompt.fillDeclarations.enabled": "boolean",
58+
"completion.prompt.fillDeclarations.maxSnippets": "number",
59+
"completion.prompt.fillDeclarations.maxChars": "number",
60+
"completion.prompt.collectSnippetsFromRecentChangedFiles": "object",
61+
"completion.prompt.collectSnippetsFromRecentChangedFiles.enabled": "boolean",
62+
"completion.prompt.collectSnippetsFromRecentChangedFiles.maxSnippets": "number",
63+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing": "object",
64+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.checkingChangesInterval": "number",
65+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.changesDebouncingInterval": "number",
66+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.prefixLines": "boolean",
67+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.suffixLines": "number",
68+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.maxChunks": "number",
69+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.chunkSize": "number",
70+
"completion.prompt.collectSnippetsFromRecentChangedFiles.indexing.overlapLines": "number",
6071
"completion.prompt.clipboard": "object",
6172
"completion.prompt.clipboard.minChars": "number",
6273
"completion.prompt.clipboard.maxChars": "number",

‎clients/tabby-agent/src/types/tabbyApi.d.ts‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ export interface components {
151151
/**
152152
* @description Filepath of the file where the snippet is from.
153153
* - When the file belongs to the same workspace as the current file,
154-
* this is a relative filepath, that has the same root as the current file.
154+
* this is a relative filepath, use the same rule as [Segments::filepath].
155155
* - When the file located outside the workspace, such as in a dependency package,
156156
* this is a file URI with an absolute filepath.
157157
*/
@@ -212,8 +212,8 @@ export interface components {
212212
suffix?: string | null;
213213
/**
214214
* @description The relative path of the file that is being edited.
215-
* - When `git_url` is set, this is the path of the file in the git repository.
216-
* - When `git_url` is empty, this is the path of the file in the workspace.
215+
* - When [Segments::git_url] is set, this is the path of the file in the git repository.
216+
* - When [Segments::git_url] is empty, this is the path of the file in the workspace.
217217
*/
218218
filepath?: string | null;
219219
/**
@@ -223,10 +223,23 @@ export interface components {
223223
*/
224224
git_url?: string | null;
225225
/**
226-
* @description The relevant declaration code snippets provided by editor.
227-
* It'll contains declarations extracted from `prefix` segments using LSP.
226+
* @description The relevant declaration code snippets provided by the editor's LSP,
227+
* contain declarations of symbols extracted from [Segments::prefix].
228228
*/
229229
declarations?: components["schemas"]["Declaration"][] | null;
230+
/**
231+
* @description The relevant code snippets extracted from recently edited files.
232+
* These snippets are selected from candidates found within code chunks
233+
* based on the edited location.
234+
* The current editing file is excluded from the search candidates.
235+
*
236+
* When provided alongside [Segments::declarations], the snippets have
237+
* already been deduplicated to ensure no duplication with entries
238+
* in [Segments::declarations].
239+
*
240+
* Sorted in descending order of [Snippet::score].
241+
*/
242+
relevant_snippets_from_changed_files?: components["schemas"]["Snippet"][] | null;
230243
/** @description Clipboard content when requesting code completion. */
231244
clipboard?: string | null;
232245
};

‎clients/vscode/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,7 @@
236236
"typescript": "^5.3.2"
237237
},
238238
"dependencies": {
239+
"@orama/orama": "^2.0.15",
239240
"@xstate/fsm": "^2.0.1",
240241
"tabby-agent": "1.5.0-dev"
241242
}

0 commit comments

Comments
 (0)