Gottfrieds_Nackenbox_GmbH/US-Entry-Coach
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ES LEBT!

Browse Source
This commit is contained in:
Simon Lübeß
2025-05-20 16:58:17 +02:00
parent e2764bd984
commit aeb0b87351
8 changed files with 584 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
USEntryCoach.Server/Program.cs
View File

@ -1,3 +1,10 @@
using System.ClientModel;
using System.Diagnostics;
using System.Text.Json.Nodes;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using OpenAI.Models;
using OpenAI.RealtimeConversation;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
// Add services to the container. // Add services to the container.
@ -17,30 +24,63 @@ if (app.Environment.IsDevelopment())
app.UseHttpsRedirection(); app.UseHttpsRedirection();
var summaries = new[] HttpClient client = new();
{ string? apiKey = app.Configuration.GetValue<string>("API:OpenAI");
"Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" client.DefaultRequestHeaders.Clear();
}; client.DefaultRequestHeaders.Add("Authorization", $"Bearer {apiKey}");
app.MapGet("/weatherforecast", () => app.MapGet("/ephemeral_token", async () =>
{ {
var forecast = Enumerable.Range(1, 5).Select(index => if (apiKey == null)
new WeatherForecast throw new Exception("API key not set");
(
DateOnly.FromDateTime(DateTime.Now.AddDays(index)), #pragma warning disable OPENAI002
Random.Shared.Next(-20, 55),
summaries[Random.Shared.Next(summaries.Length)] var options = new
)) {
.ToArray(); model = "gpt-4o-realtime-preview",
return forecast; modalities = new []{"audio", "text"},//ConversationContentModalities.Audio | ConversationContentModalities.Text,
}) voice = "ballad",//ConversationVoice.Ballad,
.WithName("GetWeatherForecast"); // TODO: Vernünfigte Werte suchen
//turn_detection = new {
//}//ConversationTurnDetectionOptions.CreateServerVoiceActivityTurnDetectionOptions(0.5f, TimeSpan.FromMilliseconds(300), TimeSpan.FromMilliseconds(500), true)
};
#pragma warning restore OPENAI002
try
{
JsonContent content = JsonContent.Create(options);
HttpResponseMessage response = await client.PostAsync("https://api.openai.com/v1/realtime/sessions", content);
Console.WriteLine($"Failed to create session: {response.RequestMessage.Content.ReadAsStringAsync().Result}");
if (response.IsSuccessStatusCode)
{
var v = await response.Content.ReadFromJsonAsync<JsonObject>();
string? ephemeralToken = v?["client_secret"]?["value"]?.GetValue<string>();
double? expiresAt = v?["client_secret"]?["expires_at"]?.GetValue<double>();
return
new {
ephemeralToken,
expiresAt,
};
}
else
{
Console.WriteLine($"Failed to create session: {response}");
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
return null;
}).WithName("GetEphemeralToken");
app.MapFallbackToFile("/index.html"); app.MapFallbackToFile("/index.html");
app.Run(); app.Run();
internal record WeatherForecast(DateOnly Date, int TemperatureC, string? Summary)
{
public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
}

6
USEntryCoach.Server/Properties/launchSettings.json
View File

@ -3,8 +3,7 @@
"http": { "http": {
"commandName": "Project", "commandName": "Project",
"environmentVariables": { "environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development", "ASPNETCORE_ENVIRONMENT": "Development"
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
}, },
"dotnetRunMessages": true, "dotnetRunMessages": true,
"applicationUrl": "http://localhost:5063" "applicationUrl": "http://localhost:5063"
@ -12,8 +11,7 @@
"https": { "https": {
"commandName": "Project", "commandName": "Project",
"environmentVariables": { "environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development", "ASPNETCORE_ENVIRONMENT": "Development"
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
}, },
"dotnetRunMessages": true, "dotnetRunMessages": true,
"applicationUrl": "https://localhost:7085;http://localhost:5063" "applicationUrl": "https://localhost:7085;http://localhost:5063"

2
USEntryCoach.Server/USEntryCoach.Server.csproj
View File

@ -9,6 +9,7 @@
<SpaRoot>..\usentrycoach.client</SpaRoot> <SpaRoot>..\usentrycoach.client</SpaRoot>
<SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand> <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
<SpaProxyServerUrl>https://localhost:54044</SpaProxyServerUrl> <SpaProxyServerUrl>https://localhost:54044</SpaProxyServerUrl>
<LangVersion>13</LangVersion>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
@ -17,6 +18,7 @@
<Version>9.*-*</Version> <Version>9.*-*</Version>
</PackageReference> </PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" /> <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
<PackageReference Include="OpenAI" Version="2.2.0-beta.4" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

2
USEntryCoach.Server/USEntryCoach.Server.http
View File

@ -1,6 +1,6 @@
@USEntryCoach.Server_HostAddress = http://localhost:5063 @USEntryCoach.Server_HostAddress = http://localhost:5063
GET {{USEntryCoach.Server_HostAddress}}/weatherforecast/ GET {{USEntryCoach.Server_HostAddress}}/ephemeraltoken/
Accept: application/json Accept: application/json
### ###

5
USEntryCoach.Server/appsettings.json
View File

@ -5,5 +5,8 @@
"Microsoft.AspNetCore": "Warning" "Microsoft.AspNetCore": "Warning"
} }
}, },
"AllowedHosts": "*" "AllowedHosts": "*",
"API": {
"OpenAI": "[NO_API_KEY]"
}
} }

451
usentrycoach.client/src/App.tsx
View File

@ -1,5 +1,6 @@
import { useEffect, useState } from 'react'; import { useEffect, useRef, useState } from 'react';
import './App.css'; import './App.css';
import {ChatControl} from "./ChatClient/ChatControl.tsx";
interface Forecast { interface Forecast {
date: string; date: string;
@ -8,51 +9,407 @@ interface Forecast {
summary: string; summary: string;
} }
function App() { interface OpenAiToken {
const [forecasts, setForecasts] = useState<Forecast[]>(); ephemeralToken: string;
expiresAt: number;
useEffect(() => {
populateWeatherData();
}, []);
const contents = forecasts === undefined
? <p><em>Loading... Please refresh once the ASP.NET backend has started. See <a href="https://aka.ms/jspsintegrationreact">https://aka.ms/jspsintegrationreact</a> for more details.</em></p>
: <table className="table table-striped" aria-labelledby="tableLabel">
<thead>
<tr>
<th>Date</th>
<th>Temp. (C)</th>
<th>Temp. (F)</th>
<th>Summary</th>
</tr>
</thead>
<tbody>
{forecasts.map(forecast =>
<tr key={forecast.date}>
<td>{forecast.date}</td>
<td>{forecast.temperatureC}</td>
<td>{forecast.temperatureF}</td>
<td>{forecast.summary}</td>
</tr>
)}
</tbody>
</table>;
return (
<div>
<h1 id="tableLabel">Weather forecast</h1>
<p>This component demonstrates fetching data from the server.</p>
{contents}
</div>
);
async function populateWeatherData() {
const response = await fetch('weatherforecast');
if (response.ok) {
const data = await response.json();
setForecasts(data);
}
}
} }
export default App;
export default function App() {
return <ChatControl/>
// const [isSessionActive, setIsSessionActive] = useState(false);
// const [events, setEvents] = useState([]);
// const [dataChannel, setDataChannel] = useState<RTCDataChannel | null>(null);
// const peerConnection = useRef<RTCPeerConnection>(null);
// const audioElement = useRef<HTMLAudioElement>(null);
//
// async function startSession() {
// // Get a session token for OpenAI Realtime API
// const tokenResponse = await fetch("/ephemeral_token");
// const data: OpenAiToken = await tokenResponse.json();
// const EPHEMERAL_KEY = data.ephemeralToken;
//
// // Create a peer connection
// const pc = new RTCPeerConnection();
//
// // Set up to play remote audio from the model
// audioElement.current = document.createElement("audio");
// audioElement.current.autoplay = true;
// pc.ontrack = (e) => {
// if (audioElement?.current !== null) {
// audioElement.current.srcObject = e.streams[0]
// }
// };
//
// // Add local audio track for microphone input in the browser
// const ms = await navigator.mediaDevices.getUserMedia({
// audio: true,
// });
// pc.addTrack(ms.getTracks()[0]);
//
// // Set up data channel for sending and receiving events
// const dc = pc.createDataChannel("oai-events");
// setDataChannel(dc);
//
// // Start the session using the Session Description Protocol (SDP)
// const offer = await pc.createOffer();
// await pc.setLocalDescription(offer);
//
// const baseUrl = "https://api.openai.com/v1/realtime";
// const model = "gpt-4o-realtime-preview-2024-12-17";
// const sdpResponse = await fetch(`${baseUrl}?model=${model}`, {
// method: "POST",
// body: offer.sdp,
// headers: {
// Authorization: `Bearer ${EPHEMERAL_KEY}`,
// "Content-Type": "application/sdp",
// },
// });
//
// const answer: RTCSessionDescriptionInit = {
// type: "answer",
// sdp: await sdpResponse.text(),
// };
// await pc.setRemoteDescription(answer);
//
// peerConnection.current = pc;
// }
//
// // Stop current session, clean up peer connection and data channel
// function stopSession() {
// if (dataChannel) {
// dataChannel.close();
// }
//
// if (peerConnection?.current !== null)
// {
// peerConnection.current.getSenders().forEach((sender) => {
// if (sender.track) {
// sender.track.stop();
// }
// });
//
// if (peerConnection.current) {
// peerConnection.current.close();
// }
// }
//
// setIsSessionActive(false);
// setDataChannel(null);
// peerConnection.current = null;
// }
//
// // Send a message to the model
// function sendClientEvent(message) {
// if (dataChannel) {
// const timestamp = new Date().toLocaleTimeString();
// message.event_id = message.event_id || crypto.randomUUID();
//
// // send event before setting timestamp since the backend peer doesn't expect this field
// dataChannel.send(JSON.stringify(message));
//
// // if guard just in case the timestamp exists by miracle
// if (!message.timestamp) {
// message.timestamp = timestamp;
// }
// setEvents((prev) => [message, ...prev]);
// } else {
// console.error(
// "Failed to send message - no data channel available",
// message,
// );
// }
// }
//
// // Send a text message to the model
// function sendTextMessage(message) {
// const event = {
// type: "conversation.item.create",
// item: {
// type: "message",
// role: "user",
// content: [
// {
// type: "input_text",
// text: message,
// },
// ],
// },
// };
//
// sendClientEvent(event);
// sendClientEvent({ type: "response.create" });
// }
//
// // Attach event listeners to the data channel when a new one is created
// useEffect(() => {
// if (dataChannel) {
// // Append new server events to the list
// dataChannel.addEventListener("message", (e) => {
// const event = JSON.parse(e.data);
// if (!event.timestamp) {
// event.timestamp = new Date().toLocaleTimeString();
// }
//
// setEvents((prev) => [event, ...prev]);
// });
//
// // Set session active when the data channel is opened
// dataChannel.addEventListener("open", () => {
// setIsSessionActive(true);
// setEvents([]);
// });
// }
// }, [dataChannel]);
//
// return (
// <>
// <nav className="absolute top-0 left-0 right-0 h-16 flex items-center">
// <div className="flex items-center gap-4 w-full m-4 pb-2 border-0 border-b border-solid border-gray-200">
// <img style={{ width: "24px" }} src={logo} />
// <h1>realtime console</h1>
// </div>
// </nav>
// <main className="absolute top-16 left-0 right-0 bottom-0">
// <section className="absolute top-0 left-0 right-[380px] bottom-0 flex">
// <section className="absolute top-0 left-0 right-0 bottom-32 px-4 overflow-y-auto">
// <EventLog events={events} />
// </section>
// <section className="absolute h-32 left-0 right-0 bottom-0 p-4">
// <SessionControls
// startSession={startSession}
// stopSession={stopSession}
// sendClientEvent={sendClientEvent}
// sendTextMessage={sendTextMessage}
// events={events}
// isSessionActive={isSessionActive}
// />
// </section>
// </section>
// <section className="absolute top-0 w-[380px] right-0 bottom-0 p-4 pt-0 overflow-y-auto">
// <ToolPanel
// sendClientEvent={sendClientEvent}
// sendTextMessage={sendTextMessage}
// events={events}
// isSessionActive={isSessionActive}
// />
// </section>
// </main>
// </>
// );
}
// export default function App() {
// const [openAiToken, setOpenAiToken] = useState<OpenAiToken | null>();
//
// useEffect(() => {
// get_the_fucking_token_already();
// }, []);
//
// const contents = openAiToken === undefined
// ? <p>Connecting to Open AI...</p>
// : <div>
// {
// openAiToken === null
// ? <p>Failed to connect to OpenAI, please reload the page.</p>
// : <p>
// {openAiToken.ephemeralToken}<br/>
// {new Date(openAiToken.expiresAt * 1000).toLocaleString()}
// </p>
// }
// </div>;
// return (
// <div>
// <h1 id="tableLabel">Weather forecast</h1>
// <p>This component demonstrates fetching data from the server.</p>
// {contents}
// </div>
// );
//
// async function get_the_fucking_token_already() {
// const response = await fetch('ephemeral_token');
// if (response.ok) {
// const data = await response.json();
// setOpenAiToken(data);
// }
// }
// }
//
// function Chatter({ openAiToken }: { openAiToken: OpenAiToken }) {
// const [isSessionActive, setIsSessionActive] = useState(false);
// const [events, setEvents] = useState([]);
// const [dataChannel, setDataChannel] = useState(null);
// const peerConnection = useRef(null);
// const audioElement = useRef(null);
//
// async function startSession() {
// // Get a session token for OpenAI Realtime API
// const tokenResponse = await fetch("/token");
// const data = await tokenResponse.json();
// const EPHEMERAL_KEY = data.client_secret.value;
//
// // Create a peer connection
// const pc = new RTCPeerConnection();
//
// // Set up to play remote audio from the model
// audioElement.current = document.createElement("audio");
// audioElement.current.autoplay = true;
// pc.ontrack = (e) => (audioElement.current.srcObject = e.streams[0]);
//
// // Add local audio track for microphone input in the browser
// const ms = await navigator.mediaDevices.getUserMedia({
// audio: true,
// });
// pc.addTrack(ms.getTracks()[0]);
//
// // Set up data channel for sending and receiving events
// const dc = pc.createDataChannel("oai-events");
// setDataChannel(dc);
//
// // Start the session using the Session Description Protocol (SDP)
// const offer = await pc.createOffer();
// await pc.setLocalDescription(offer);
//
// const baseUrl = "https://api.openai.com/v1/realtime";
// const model = "gpt-4o-realtime-preview-2024-12-17";
// const sdpResponse = await fetch(`${baseUrl}?model=${model}`, {
// method: "POST",
// body: offer.sdp,
// headers: {
// Authorization: `Bearer ${EPHEMERAL_KEY}`,
// "Content-Type": "application/sdp",
// },
// });
//
// const answer = {
// type: "answer",
// sdp: await sdpResponse.text(),
// };
// await pc.setRemoteDescription(answer);
//
// peerConnection.current = pc;
// }
//
// // Stop current session, clean up peer connection and data channel
// function stopSession() {
// if (dataChannel) {
// dataChannel.close();
// }
//
// peerConnection.current.getSenders().forEach((sender) => {
// if (sender.track) {
// sender.track.stop();
// }
// });
//
// if (peerConnection.current) {
// peerConnection.current.close();
// }
//
// setIsSessionActive(false);
// setDataChannel(null);
// peerConnection.current = null;
// }
//
// // Send a message to the model
// function sendClientEvent(message) {
// if (dataChannel) {
// const timestamp = new Date().toLocaleTimeString();
// message.event_id = message.event_id || crypto.randomUUID();
//
// // send event before setting timestamp since the backend peer doesn't expect this field
// dataChannel.send(JSON.stringify(message));
//
// // if guard just in case the timestamp exists by miracle
// if (!message.timestamp) {
// message.timestamp = timestamp;
// }
// setEvents((prev) => [message, ...prev]);
// } else {
// console.error(
// "Failed to send message - no data channel available",
// message,
// );
// }
// }
//
// // Send a text message to the model
// function sendTextMessage(message) {
// const event = {
// type: "conversation.item.create",
// item: {
// type: "message",
// role: "user",
// content: [
// {
// type: "input_text",
// text: message,
// },
// ],
// },
// };
//
// sendClientEvent(event);
// sendClientEvent({ type: "response.create" });
// }
//
// // Attach event listeners to the data channel when a new one is created
// useEffect(() => {
// if (dataChannel) {
// // Append new server events to the list
// dataChannel.addEventListener("message", (e) => {
// const event = JSON.parse(e.data);
// if (!event.timestamp) {
// event.timestamp = new Date().toLocaleTimeString();
// }
//
// setEvents((prev) => [event, ...prev]);
// });
//
// // Set session active when the data channel is opened
// dataChannel.addEventListener("open", () => {
// setIsSessionActive(true);
// setEvents([]);
// });
// }
// }, [dataChannel]);
//
// return (
// <>
// <nav className="absolute top-0 left-0 right-0 h-16 flex items-center">
// <div className="flex items-center gap-4 w-full m-4 pb-2 border-0 border-b border-solid border-gray-200">
// <img style={{ width: "24px" }} src={logo} />
// <h1>realtime console</h1>
// </div>
// </nav>
// <main className="absolute top-16 left-0 right-0 bottom-0">
// <section className="absolute top-0 left-0 right-[380px] bottom-0 flex">
// <section className="absolute top-0 left-0 right-0 bottom-32 px-4 overflow-y-auto">
// <EventLog events={events} />
// </section>
// <section className="absolute h-32 left-0 right-0 bottom-0 p-4">
// <SessionControls
// startSession={startSession}
// stopSession={stopSession}
// sendClientEvent={sendClientEvent}
// sendTextMessage={sendTextMessage}
// events={events}
// isSessionActive={isSessionActive}
// />
// </section>
// </section>
// <section className="absolute top-0 w-[380px] right-0 bottom-0 p-4 pt-0 overflow-y-auto">
// <ToolPanel
// sendClientEvent={sendClientEvent}
// sendTextMessage={sendTextMessage}
// events={events}
// isSessionActive={isSessionActive}
// />
// </section>
// </main>
// </>
// );
// }

109
usentrycoach.client/src/ChatClient/ChatControl.tsx Normal file
View File

@ -0,0 +1,109 @@
import {useState, useRef, useEffect} from 'react';
function SessionActive({stopSession} : {stopSession: () => void}) {
return <button onClick={stopSession}>Stop session</button>
}
function SessionStopped({startSession} : {startSession: () => void}) {
return <button onClick={startSession}>Start session</button>
}
function SessionControl( { isSessionActive, startSession, stopSession }: { isSessionActive: boolean, startSession : () => void, stopSession : () => void })
{
return (isSessionActive
? <SessionActive stopSession={stopSession}/>
: <SessionStopped startSession={startSession}/>);
}
interface OpenAiToken {
ephemeralToken: string;
expiresAt: number;
}
export function ChatControl() {
const [isSessionActive, setSessionActive] = useState<boolean>(false);
const [dataChannel, setDataChannel] = useState<RTCDataChannel | null>(null);
const audioElement = useRef<HTMLAudioElement>(null);
const peerConnection = useRef<RTCPeerConnection>(null);
async function StartSession() {
// Get a session token for OpenAI Realtime API
const response = await fetch('ephemeral_token');
if (!response.ok) {
throw new Error(response.statusText);
}
const data: OpenAiToken = await response.json();
const ephemeralToken = data.ephemeralToken;
// Create a peer connection
const pc = new RTCPeerConnection();
// Set up to play remote audio from the model
audioElement.current = document.createElement("audio");
audioElement.current.autoplay = true;
pc.ontrack = (e) => {
if (audioElement.current !== null)
{
audioElement.current.srcObject = e.streams[0];
}
}
// Add local audio track for microphone input in the browser
const microphoneStream = await navigator.mediaDevices.getUserMedia({audio: true});
pc.addTrack(microphoneStream.getTracks()[0]);
// Set up data channel for sending and receving events
const dc = pc.createDataChannel("oai-events");
setDataChannel(dc)
// Start the session using the Session Description Protocol (SDP)
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// Start Realtime Session
const baseUrl = "https://api.openai.com/v1/realtime";
const model = "gpt-4o-realtime-preview";
const sdpResponse = await fetch(`${baseUrl}?model=${model}`, {
method: "POST",
body: offer.sdp,
headers: {
Authorization: `Bearer ${ephemeralToken}`,
"Content-Type": "application/sdp",
},
});
const answer:RTCSessionDescriptionInit = {
type: "answer",
sdp: await sdpResponse.text(),
};
await pc.setRemoteDescription(answer);
peerConnection.current = pc;
}
function stopSession() {
if (dataChannel) {
dataChannel.close();
setDataChannel(null);
}
if (peerConnection.current !== null)
{
peerConnection.current.close();
peerConnection.current = null;
}
setSessionActive(false);
}
useEffect(() => {
if (dataChannel) {
dataChannel.addEventListener("open", () => {
setSessionActive(true);
});
}
}, [dataChannel])
return <SessionControl isSessionActive={isSessionActive} startSession={StartSession} stopSession={stopSession}/>;
}

2
usentrycoach.client/vite.config.ts
View File

@ -47,7 +47,7 @@ export default defineConfig({
}, },
server: { server: {
proxy: { proxy: {
'^/weatherforecast': { '^/ephemeral_token': {
target, target,
secure: false secure: false
} }