add language detection
This commit is contained in:
parent
4cfc368f96
commit
134aa4fc6a
@ -11,9 +11,9 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@radix-ui/react-slot": "^1.0.2",
|
"@radix-ui/react-slot": "^1.0.2",
|
||||||
"@types/react-syntax-highlighter": "^15.5.11",
|
"@types/react-syntax-highlighter": "^15.5.11",
|
||||||
|
"@vscode/vscode-languagedetection": "^1.0.22",
|
||||||
"class-variance-authority": "^0.7.0",
|
"class-variance-authority": "^0.7.0",
|
||||||
"clsx": "^2.1.0",
|
"clsx": "^2.1.0",
|
||||||
"lang-detector": "^1.0.6",
|
|
||||||
"lucide-react": "^0.372.0",
|
"lucide-react": "^0.372.0",
|
||||||
"moment": "^2.30.1",
|
"moment": "^2.30.1",
|
||||||
"next": "14.2.2",
|
"next": "14.2.2",
|
||||||
|
21
pnpm-lock.yaml
generated
21
pnpm-lock.yaml
generated
@ -11,15 +11,15 @@ dependencies:
|
|||||||
'@types/react-syntax-highlighter':
|
'@types/react-syntax-highlighter':
|
||||||
specifier: ^15.5.11
|
specifier: ^15.5.11
|
||||||
version: 15.5.11
|
version: 15.5.11
|
||||||
|
'@vscode/vscode-languagedetection':
|
||||||
|
specifier: ^1.0.22
|
||||||
|
version: 1.0.22
|
||||||
class-variance-authority:
|
class-variance-authority:
|
||||||
specifier: ^0.7.0
|
specifier: ^0.7.0
|
||||||
version: 0.7.0
|
version: 0.7.0
|
||||||
clsx:
|
clsx:
|
||||||
specifier: ^2.1.0
|
specifier: ^2.1.0
|
||||||
version: 2.1.0
|
version: 2.1.0
|
||||||
lang-detector:
|
|
||||||
specifier: ^1.0.6
|
|
||||||
version: 1.0.6
|
|
||||||
lucide-react:
|
lucide-react:
|
||||||
specifier: ^0.372.0
|
specifier: ^0.372.0
|
||||||
version: 0.372.0(react@18.2.0)
|
version: 0.372.0(react@18.2.0)
|
||||||
@ -452,6 +452,11 @@ packages:
|
|||||||
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
|
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/@vscode/vscode-languagedetection@1.0.22:
|
||||||
|
resolution: {integrity: sha512-rQ/BgMyLuIXSmbA0MSkIPHtcOw14QkeDbAq19sjvaS9LTRr905yij0S8lsyqN5JgOsbtIx7pAcyOxFMzPmqhZQ==}
|
||||||
|
hasBin: true
|
||||||
|
dev: false
|
||||||
|
|
||||||
/acorn-jsx@5.3.2(acorn@8.11.3):
|
/acorn-jsx@5.3.2(acorn@8.11.3):
|
||||||
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
|
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
@ -1892,12 +1897,6 @@ packages:
|
|||||||
json-buffer: 3.0.1
|
json-buffer: 3.0.1
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/lang-detector@1.0.6:
|
|
||||||
resolution: {integrity: sha512-YRt/OzgBlrCf8AGiFLWXuqbvEuOI9xHJQe73iKRGGfOZrylDyBAMUWWhWwVdCQxxX0yN5V52PDypqXnlVb4C/w==}
|
|
||||||
dependencies:
|
|
||||||
underscore: 1.13.6
|
|
||||||
dev: false
|
|
||||||
|
|
||||||
/language-subtag-registry@0.3.22:
|
/language-subtag-registry@0.3.22:
|
||||||
resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==}
|
resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==}
|
||||||
dev: true
|
dev: true
|
||||||
@ -2887,10 +2886,6 @@ packages:
|
|||||||
which-boxed-primitive: 1.0.2
|
which-boxed-primitive: 1.0.2
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/underscore@1.13.6:
|
|
||||||
resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
|
|
||||||
dev: false
|
|
||||||
|
|
||||||
/undici-types@5.26.5:
|
/undici-types@5.26.5:
|
||||||
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
|
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
|
||||||
dev: true
|
dev: true
|
||||||
|
@ -1 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
|
Before Width: | Height: | Size: 1.3 KiB |
@ -1 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>
|
|
Before Width: | Height: | Size: 629 B |
@ -1,9 +1,10 @@
|
|||||||
import { ReactElement } from "react";
|
import { cache, ReactElement } from "react";
|
||||||
import { ActionMenu } from "@/app/components/action-menu";
|
import { ActionMenu } from "@/app/components/action-menu";
|
||||||
import { Metadata } from "next";
|
import { Metadata } from "next";
|
||||||
import moment from "moment";
|
import moment from "moment";
|
||||||
import { notFound } from "next/navigation";
|
import { notFound } from "next/navigation";
|
||||||
import { CodeBlock } from "@/app/components/codeBlock";
|
import { CodeBlock } from "@/app/components/codeBlock";
|
||||||
|
import { detectLanguage } from "@/app/common/lang-detection/detection";
|
||||||
|
|
||||||
type PasteProps = {
|
type PasteProps = {
|
||||||
params: {
|
params: {
|
||||||
@ -21,25 +22,14 @@ type Paste = {
|
|||||||
* The date the paste was created.
|
* The date the paste was created.
|
||||||
*/
|
*/
|
||||||
created: number;
|
created: number;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The detected language of the paste.
|
||||||
|
*/
|
||||||
|
language: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export async function generateMetadata({
|
const getPaste = cache(async (id: string) => {
|
||||||
params: { id },
|
|
||||||
}: PasteProps): Promise<Metadata> {
|
|
||||||
const data: Paste | undefined = await getData(id);
|
|
||||||
if (data == undefined) {
|
|
||||||
return {
|
|
||||||
description: "Not found",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
title: `Paste - ${id}`,
|
|
||||||
description: `Created: ${moment(data.created)}\n\nClick to view the paste.`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getData(id: string): Promise<Paste | undefined> {
|
|
||||||
const response: Response = await fetch(
|
const response: Response = await fetch(
|
||||||
`${process.env.NEXT_PUBLIC_API_ENDPOINT}/${id}`,
|
`${process.env.NEXT_PUBLIC_API_ENDPOINT}/${id}`,
|
||||||
{
|
{
|
||||||
@ -53,13 +43,33 @@ async function getData(id: string): Promise<Paste | undefined> {
|
|||||||
if (json.code && json.message) {
|
if (json.code && json.message) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
return json as Paste;
|
return {
|
||||||
|
content: json.content,
|
||||||
|
created: json.created,
|
||||||
|
language: await detectLanguage(json.content),
|
||||||
|
};
|
||||||
|
}) as (id: string) => Promise<Paste | undefined>;
|
||||||
|
|
||||||
|
export async function generateMetadata({
|
||||||
|
params: { id },
|
||||||
|
}: PasteProps): Promise<Metadata> {
|
||||||
|
const data: Paste | undefined = await getPaste(id);
|
||||||
|
if (data == undefined) {
|
||||||
|
return {
|
||||||
|
description: "Not found",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: `Paste - ${id}`,
|
||||||
|
description: `Created: ${moment(data.created)}\n\nClick to view the paste.`,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export default async function Paste({
|
export default async function Paste({
|
||||||
params: { id },
|
params: { id },
|
||||||
}: PasteProps): Promise<ReactElement> {
|
}: PasteProps): Promise<ReactElement> {
|
||||||
const data: Paste | undefined = await getData(id);
|
const data: Paste | undefined = await getPaste(id);
|
||||||
|
|
||||||
if (data == undefined) {
|
if (data == undefined) {
|
||||||
return notFound();
|
return notFound();
|
||||||
@ -67,7 +77,10 @@ export default async function Paste({
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="relative">
|
<div className="relative">
|
||||||
<ActionMenu />
|
<div className="absolute top-0 right-0 flex flex-col items-end mx-3 mt-2">
|
||||||
|
<ActionMenu />
|
||||||
|
<p>{data.language}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="p-1 hljs !bg-transparent text-sm">
|
<div className="p-1 hljs !bg-transparent text-sm">
|
||||||
<CodeBlock code={data.content} />
|
<CodeBlock code={data.content} />
|
||||||
|
@ -45,9 +45,11 @@ export default function Home(): ReactElement {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<ActionMenu>
|
<div className="absolute top-0 right-0 mx-3 mt-2">
|
||||||
<Button onClick={() => createPaste()}>Save</Button>
|
<ActionMenu>
|
||||||
</ActionMenu>
|
<Button onClick={() => createPaste()}>Save</Button>
|
||||||
|
</ActionMenu>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
25
src/app/common/lang-detection/detection.ts
Normal file
25
src/app/common/lang-detection/detection.ts
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
import { ModelOperations } from "@vscode/vscode-languagedetection";
|
||||||
|
import fs from "node:fs";
|
||||||
|
|
||||||
|
const dir = `${process.cwd()}/src/app/common/lang-detection`;
|
||||||
|
|
||||||
|
const modelOperations = new ModelOperations({
|
||||||
|
modelJsonLoaderFunc: async () => {
|
||||||
|
const response = await fs.promises.readFile(dir + "/model.json");
|
||||||
|
return JSON.parse(response.toString());
|
||||||
|
},
|
||||||
|
weightsLoaderFunc: async () => {
|
||||||
|
const response = await fs.promises.readFile(dir + "/weights.bin");
|
||||||
|
return response.buffer;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detects the programming language of the given content.
|
||||||
|
*
|
||||||
|
* @param content The content to detect the language of.
|
||||||
|
*/
|
||||||
|
export async function detectLanguage(content: string) {
|
||||||
|
const languages = await modelOperations.runModel(content);
|
||||||
|
return languages.length > 0 ? languages[0].languageId : "Text";
|
||||||
|
}
|
1
src/app/common/lang-detection/model.json
Normal file
1
src/app/common/lang-detection/model.json
Normal file
File diff suppressed because one or more lines are too long
BIN
src/app/common/lang-detection/weights.bin
Normal file
BIN
src/app/common/lang-detection/weights.bin
Normal file
Binary file not shown.
@ -8,7 +8,7 @@ type ActionMenuProps = {
|
|||||||
|
|
||||||
export function ActionMenu({ children }: ActionMenuProps) {
|
export function ActionMenu({ children }: ActionMenuProps) {
|
||||||
return (
|
return (
|
||||||
<div className="absolute top-0 right-0 flex items-center mx-3 mt-2 bg-secondary rounded-md p-2 gap-2">
|
<div className="flex items-center bg-secondary rounded-md p-2 gap-2">
|
||||||
{children}
|
{children}
|
||||||
<Link href={"/"}>
|
<Link href={"/"}>
|
||||||
<Button>New</Button>
|
<Button>New</Button>
|
||||||
|
Reference in New Issue
Block a user