add language detection
Some checks failed
Publish Docker Image / docker (ubuntu-latest) (push) Waiting to run
Deploy App / docker (ubuntu-latest) (push) Has been cancelled

This commit is contained in:
Lee 2024-04-23 17:22:29 +01:00
parent 4cfc368f96
commit 134aa4fc6a
10 changed files with 75 additions and 41 deletions

@ -11,9 +11,9 @@
"dependencies": {
"@radix-ui/react-slot": "^1.0.2",
"@types/react-syntax-highlighter": "^15.5.11",
"@vscode/vscode-languagedetection": "^1.0.22",
"class-variance-authority": "^0.7.0",
"clsx": "^2.1.0",
"lang-detector": "^1.0.6",
"lucide-react": "^0.372.0",
"moment": "^2.30.1",
"next": "14.2.2",

21
pnpm-lock.yaml generated

@ -11,15 +11,15 @@ dependencies:
'@types/react-syntax-highlighter':
specifier: ^15.5.11
version: 15.5.11
'@vscode/vscode-languagedetection':
specifier: ^1.0.22
version: 1.0.22
class-variance-authority:
specifier: ^0.7.0
version: 0.7.0
clsx:
specifier: ^2.1.0
version: 2.1.0
lang-detector:
specifier: ^1.0.6
version: 1.0.6
lucide-react:
specifier: ^0.372.0
version: 0.372.0(react@18.2.0)
@ -452,6 +452,11 @@ packages:
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
dev: true
/@vscode/vscode-languagedetection@1.0.22:
resolution: {integrity: sha512-rQ/BgMyLuIXSmbA0MSkIPHtcOw14QkeDbAq19sjvaS9LTRr905yij0S8lsyqN5JgOsbtIx7pAcyOxFMzPmqhZQ==}
hasBin: true
dev: false
/acorn-jsx@5.3.2(acorn@8.11.3):
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
peerDependencies:
@ -1892,12 +1897,6 @@ packages:
json-buffer: 3.0.1
dev: true
/lang-detector@1.0.6:
resolution: {integrity: sha512-YRt/OzgBlrCf8AGiFLWXuqbvEuOI9xHJQe73iKRGGfOZrylDyBAMUWWhWwVdCQxxX0yN5V52PDypqXnlVb4C/w==}
dependencies:
underscore: 1.13.6
dev: false
/language-subtag-registry@0.3.22:
resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==}
dev: true
@ -2887,10 +2886,6 @@ packages:
which-boxed-primitive: 1.0.2
dev: true
/underscore@1.13.6:
resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
dev: false
/undici-types@5.26.5:
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
dev: true

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

Before

Width:  |  Height:  |  Size: 629 B

@ -1,9 +1,10 @@
import { ReactElement } from "react";
import { cache, ReactElement } from "react";
import { ActionMenu } from "@/app/components/action-menu";
import { Metadata } from "next";
import moment from "moment";
import { notFound } from "next/navigation";
import { CodeBlock } from "@/app/components/codeBlock";
import { detectLanguage } from "@/app/common/lang-detection/detection";
type PasteProps = {
params: {
@ -21,25 +22,14 @@ type Paste = {
* The date the paste was created.
*/
created: number;
/**
* The detected language of the paste.
*/
language: string;
};
export async function generateMetadata({
params: { id },
}: PasteProps): Promise<Metadata> {
const data: Paste | undefined = await getData(id);
if (data == undefined) {
return {
description: "Not found",
};
}
return {
title: `Paste - ${id}`,
description: `Created: ${moment(data.created)}\n\nClick to view the paste.`,
};
}
async function getData(id: string): Promise<Paste | undefined> {
const getPaste = cache(async (id: string) => {
const response: Response = await fetch(
`${process.env.NEXT_PUBLIC_API_ENDPOINT}/${id}`,
{
@ -53,13 +43,33 @@ async function getData(id: string): Promise<Paste | undefined> {
if (json.code && json.message) {
return undefined;
}
return json as Paste;
return {
content: json.content,
created: json.created,
language: await detectLanguage(json.content),
};
}) as (id: string) => Promise<Paste | undefined>;
export async function generateMetadata({
params: { id },
}: PasteProps): Promise<Metadata> {
const data: Paste | undefined = await getPaste(id);
if (data == undefined) {
return {
description: "Not found",
};
}
return {
title: `Paste - ${id}`,
description: `Created: ${moment(data.created)}\n\nClick to view the paste.`,
};
}
export default async function Paste({
params: { id },
}: PasteProps): Promise<ReactElement> {
const data: Paste | undefined = await getData(id);
const data: Paste | undefined = await getPaste(id);
if (data == undefined) {
return notFound();
@ -67,7 +77,10 @@ export default async function Paste({
return (
<div className="relative">
<div className="absolute top-0 right-0 flex flex-col items-end mx-3 mt-2">
<ActionMenu />
<p>{data.language}</p>
</div>
<div className="p-1 hljs !bg-transparent text-sm">
<CodeBlock code={data.content} />

@ -45,9 +45,11 @@ export default function Home(): ReactElement {
/>
</div>
<div className="absolute top-0 right-0 mx-3 mt-2">
<ActionMenu>
<Button onClick={() => createPaste()}>Save</Button>
</ActionMenu>
</div>
</div>
);
}

@ -0,0 +1,25 @@
import { ModelOperations } from "@vscode/vscode-languagedetection";
import fs from "node:fs";
const dir = `${process.cwd()}/src/app/common/lang-detection`;
const modelOperations = new ModelOperations({
modelJsonLoaderFunc: async () => {
const response = await fs.promises.readFile(dir + "/model.json");
return JSON.parse(response.toString());
},
weightsLoaderFunc: async () => {
const response = await fs.promises.readFile(dir + "/weights.bin");
return response.buffer;
},
});
/**
* Detects the programming language of the given content.
*
* @param content The content to detect the language of.
*/
export async function detectLanguage(content: string) {
const languages = await modelOperations.runModel(content);
return languages.length > 0 ? languages[0].languageId : "Text";
}

File diff suppressed because one or more lines are too long

Binary file not shown.

@ -8,7 +8,7 @@ type ActionMenuProps = {
export function ActionMenu({ children }: ActionMenuProps) {
return (
<div className="absolute top-0 right-0 flex items-center mx-3 mt-2 bg-secondary rounded-md p-2 gap-2">
<div className="flex items-center bg-secondary rounded-md p-2 gap-2">
{children}
<Link href={"/"}>
<Button>New</Button>