index.html

<style type="text/css">
  html,
  body {
    margin: 0;
    padding: 0;
    height: 100%;
  }

  body {
    /* display: flex; */
    /* flex-direction: column;
    align-items: center;
    justify-content: center; */
  }

  #log {
    padding: 1em;
    padding-top: calc(1em + 2 * 1em + 20px);
    font-family: monospace;
  }

  #status {
    position: fixed;
    left: 0;
    top: 0;
    right: 0;
    height: 20px;
    padding: 1em;
    background: #00000015;
    font-weight: 600;
    color: #111;
    backdrop-filter: blur(5px);
    -webkit-backdrop-filter: blur(5px);
    border-bottom: 1px solid #ddd;
    font-family: "Helvetica", sans-serif;
  }

  .session {
    display: flex;
    flex-direction: column;
    margin-bottom: 1em;
    padding: 1em;
    background: #00000008;
    border-radius: 5px;
  }

  .session .costs {
    opacity: 0.3;
    font-size: 0.8em;
    order: 99;
    margin-top: 1em;
  }

  .message {
    margin-bottom: 0;
  }
</style>

<div id="status"></div>
<div id="log"></div>

<script src="https://cdn.jsdelivr.net/npm/dat.gui@0.7.9/build/dat.gui.min.js"></script>
<script src="https://cdn.athom.com/homey-api/3.6.2.js"></script>
<script type="text/javascript">
  Promise.resolve().then(async () => {
    // If not Google Chrome, don't continue
    if (!window.chrome) {
      alert('This demo requires Google Chrome.');
      return;
    }

    // https://openai.com/api/pricing/
    const COSTS_PER_TOKEN = { // in USD
      'gpt-4o': {
        input: 5.00 / 1000000,
        output: 15.00 / 1000000,
      },
      'gpt-4o-mini': {
        input: 0.150 / 1000000,
        output: 0.600 / 1000000,
      },
    }

    let user;
    let homey;
    let homeyApi;
    let devices;
    let zones;

    const deviceIdsByNumber = {};
    let state = 'idle';
    let mediaRecorder;
    let audioPlayer;

    const $status = document.getElementById('status');
    const $log = document.getElementById('log');

    // Initialize dat.gui
    const settings = {
      apiKey: new URL(window.location).searchParams.get('api_key') || localStorage.getItem('apiKey') || '',
      ttsVoice: 'nova',
      ttsModel: 'tts-1-hd',
      chatModel: 'gpt-4o', // Note: gp-4o-mini has much more incorrect results
    };

    const gui = new dat.GUI();
    gui.add(settings, 'apiKey').name('OpenAI API Key').onChange(value => {
      localStorage.setItem('apiKey', value);
    });
    gui.add(settings, 'ttsVoice', ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer']).name('TTS Voice');
    gui.add(settings, 'ttsModel', ['tts-1', 'tts-1-hd']).name('TTS Model');
    gui.add(settings, 'chatModel', ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo', 'gpt-4']).name('Chat Model');

    // Initialize Homey API
    setStatus('Connecting to Homey...');
    const athomCloudAPI = new AthomCloudAPI({
      clientId: '66b49b944233725c428a224c',
      clientSecret: 'ad12da1f42aa72f151c031b80d9447b52099f688',
      redirectUrl: window.location.origin + window.location.pathname,
    });

    const isLoggedIn = await athomCloudAPI.isLoggedIn();
    if (!isLoggedIn) {
      if (athomCloudAPI.hasAuthorizationCode()) {
        const token = await athomCloudAPI.authenticateWithAuthorizationCode();
      } else {
        window.location.href = athomCloudAPI.getLoginUrl();
        return;
      }
    }

    user = await athomCloudAPI.getAuthenticatedUser();
    logMessage(`<p>👋 Hi, ${user.firstname}! <a id="signout" href="#">Sign out »</a></p>`);

    document.getElementById('signout').addEventListener('click', async e => {
      e.preventDefault();
      await athomCloudAPI.logout();
      window.location.reload();
    });

    homey = await user.getFirstHomey();
    homeyApi = await homey.authenticate();
    logMessage(`<p>🏠 Homey: ${homey.name}</p>`);

    // Initialize Devices
    async function initDevice(device) {
      for (const capabilityId of device.capabilities) {
        await device.makeCapabilityInstance(capabilityId, () => { });
      }
    }

    await homeyApi.devices.connect();
    devices = await homeyApi.devices.getDevices();
    for (const device of Object.values(devices)) {
      await initDevice(device);
    }

    homeyApi.devices
      .on('device.create', device => initDevice(device).catch(err => console.error(err)))
      .on('device.update', device => initDevice(device).catch(err => console.error(err)));

    // Initialize Zones
    await homeyApi.zones.connect();
    zones = await homeyApi.zones.getZones();

    logMessage('<p>✅ Connected!</p>');

    logMessage('<p>Try saying:</p>');
    logMessage('<li>"Turn on the lights in the living room."</li>');
    logMessage('<li>"Turn off all lights on the first floor, but keep bedroom on."</li>');
    logMessage('<li>"Turn on Standing Light, then turn it off after 5 seconds."</li>');
    logMessage('<li>"Turn on bedroom at 2 pm."</li>');

    logMessage('<p>&nbsp;</p>');

    setStatus();

    window.addEventListener('keydown', e => {
      if (audioPlayer && audioPlayer.paused === false) {
        audioPlayer.pause();
      }

      if (e.key === ' ' && state === 'idle') {
        state = 'starting';

        // Start Listening
        Promise.resolve().then(async () => {
          const $session = document.createElement('div');
          $session.classList.add('session');
          $log.appendChild($session);

          const $sessionCosts = document.createElement('div');
          $sessionCosts.classList.add('costs');
          $session.appendChild($sessionCosts);

          function logMessage(text) {
            const $message = document.createElement('p');
            $message.classList.add('message');
            $message.innerHTML = text;
            $session.appendChild($message);
          }

          ///////////////////////
          // Step 1 — Stream Microphone to OpenAI API
          ///////////////////////
          const inputAudio = await Promise.resolve().then(async () => {
            let audioChunks = [];

            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
            mediaRecorder = new MediaRecorder(stream);

            // Initialize MediaRecorder
            mediaRecorder = new MediaRecorder(stream);
            mediaRecorder.start();

            mediaRecorder.addEventListener('start', () => {
              state = 'listening';
              setStatus('Listening...');
            });

            mediaRecorder.addEventListener('dataavailable', event => {
              if (event.data.size > 0) {
                audioChunks.push(event.data);
              }
            });

            return new Promise((resolve, reject) => {
              mediaRecorder.addEventListener('stop', () => {

                // Create Blob
                audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
                resolve(audioBlob);
              });
            });
          });

          ///////////////////////
          // Step 2 — STT
          ///////////////////////
          const inputText = await Promise.resolve().then(async () => {
            state = 'analyzing_audio';
            setStatus('Analyzing Audio...');

            // Initialize request to OpenAI API
            const formData = new FormData();
            formData.append('model', 'whisper-1');
            formData.append('file', inputAudio, 'recording.webm');

            const response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
              method: 'POST',
              headers: {
                'Authorization': `Bearer ${settings.apiKey}`,
              },
              body: formData,
            });

            if (!response.ok) {
              const body = await response.json().catch(err => {
                throw new Error(response.statusText ?? response.status);
              });
              throw new Error(body.error.message ?? response.statusText ?? response.status);
            }

            const { text } = await response.json();
            return text;
          });

          logMessage(`🎙️ ${inputText}`);

          ///////////////////////
          // Step 3 — Chat
          ///////////////////////
          const outputText = await Promise.resolve().then(async () => {
            state = 'analyzing_text';
            setStatus('Analyzing Text...');

            const response = await fetch('https://api.openai.com/v1/chat/completions', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${settings.apiKey}`,
              },
              body: JSON.stringify({
                model: settings.chatModel,
                response_format: {
                  type: 'json_object', // TODO: JSON Schema
                },
                messages: [
                  {
                    role: 'system',
                    content: 'You are a smart home assistant. You may only answer queries related to smart home, or time. You will change the state of devices, and return the state in the following JSON format: ' + JSON.stringify({
                      text: '<The textual response. Short and to the point.>',
                      actions: [
                        {
                          '<device-id>': {
                            // name: '<the name of the device>',
                            // zone: '<the zone of the device>',
                            on: '<the new value as boolean, only if changed or if set with a timer>',
                            brightness: '<the new value as number (1-100), only if changed or if set with a timer>',
                            delay: '<number, in seconds, but only if a timer has been requested>'
                          },
                        },
                      ],
                    }),
                  },
                  {
                    role: 'system',
                    content: 'The current time is ' + new Date().toLocaleTimeString(),
                  },
                  {
                    role: 'system',
                    content: 'This is the JSON state of the smart home: ' + JSON.stringify(await getOptimizedDevicesObject()),
                  },
                  {
                    role: 'user',
                    content: inputText,
                  },
                ],
              }),
            });

            if (!response.ok) {
              throw new Error(response.statusText);
            }

            const data = await response.json();
            const content = data.choices[0].message.content;
            const payload = JSON.parse(content);
            console.log(JSON.stringify(payload, null, 2));

            const costsInput = COSTS_PER_TOKEN[settings.chatModel]?.input ?? 0 * data.usage.prompt_tokens;
            const costsOutput = COSTS_PER_TOKEN[settings.chatModel]?.output ?? 0 * data.usage.completion_tokens;

            logMessage(`🤖 ${payload.text}`);
            $sessionCosts.innerHTML = `${data.usage.prompt_tokens} input + ${data.usage.completion_tokens} output = ${data.usage.total_tokens} tokens • $${costsInput} + $${costsOutput} = $${costsInput + costsOutput}`;

            for (const action of Object.values(payload.actions ?? {})) {
              for (const [deviceNumber, newState] of Object.entries(action)) {
                const deviceId = Object.keys(deviceIdsByNumber).find(deviceId => deviceIdsByNumber[deviceId] === parseInt(deviceNumber));
                if (!deviceId) continue;

                const device = devices[deviceId];
                if (!device) continue;

                const deviceZone = await device.getZone();
                const delay = newState.delay ?? 0;

                if (newState.on !== undefined) {
                  if (delay) {
                    logMessage(`⏳ ${device.name} (${deviceZone.name}): ${newState.on ? 'On' : 'Off'} in ${delay}s`);
                  } else {
                    logMessage(`💡 ${device.name} (${deviceZone.name}): ${newState.on ? 'On' : 'Off'}`);
                  }

                  setTimeout(() => {
                    device.setCapabilityValue('onoff', newState.on)
                      .catch(err => console.error(err));
                  }, delay * 1000);
                }

                if (newState.brightness !== undefined) {
                  if (delay) {
                    logMessage(`⏳ ${device.name} (${deviceZone.name}): ${newState.brightness}% in ${delay}s`);
                  } else {
                    logMessage(`💡 ${device.name} (${deviceZone.name}): ${newState.brightness}%`);
                  }

                  setTimeout(() => {
                    device.setCapabilityValue('dim', newState.brightness / 100)
                      .catch(err => console.error(err));
                  }, delay * 1000);
                }
              }
            }

            return payload.text;
          });


          ///////////////////////
          // Step 4 — TTS
          ///////////////////////
          await Promise.resolve().then(async () => {
            state = 'speaking';
            setStatus('Speaking...');

            const response = await fetch('https://api.openai.com/v1/audio/speech', {
              method: 'POST',
              headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${settings.apiKey}`
              },
              body: JSON.stringify({
                input: outputText,
                voice: settings.ttsVoice,
                model: settings.ttsModel,
              }),
            });

            if (!response.ok) {
              throw new Error(response.statusText);
            }

            // Create a new MediaSource
            const mediaSource = new MediaSource();
            audioPlayer = new Audio();
            audioPlayer.src = URL.createObjectURL(mediaSource);

            mediaSource.addEventListener('sourceopen', async () => {
              const sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');

              // Function to stream audio data into the source buffer
              async function appendStream() {
                const reader = response.body.getReader();
                const pump = async () => {
                  const { done, value } = await reader.read();
                  if (done) {
                    mediaSource.endOfStream();
                    return;
                  }
                  sourceBuffer.appendBuffer(value);
                  await new Promise(resolve => {
                    sourceBuffer.addEventListener('updateend', resolve, { once: true });
                  });
                  await pump();
                };
                await pump();
              }

              // Start streaming
              appendStream();
            });

            // Play audio as soon as data is available
            await new Promise((resolve, reject) => {
              audioPlayer.play().catch(err => console.error(err));
              audioPlayer.addEventListener('error', reject);
              audioPlayer.addEventListener('pause', () => resolve());
              audioPlayer.addEventListener('ended', () => resolve());
            });
          });

          setStatus();
          state = 'idle';
        }).catch(err => {
          logMessage(`❌ ${err.message}`);

          setStatus();
          state = 'idle';
        });
      }
    });

    window.addEventListener('keyup', e => {
      if (e.key === ' ' && state === 'listening') {
        mediaRecorder.stop();
      }
    });

    function setStatus(text = 'Hold [Space] to start talking.') {
      $status.innerText = text;
    }

    function logMessage(text) {
      const $message = document.createElement('div');
      $message.classList.add('message');
      $message.innerHTML = text;
      $log.appendChild($message);
    }

    async function getOptimizedDevicesObject() {
      const result = {};

      for (const device of Object.values(devices)) {
        if (!device.capabilitiesObj) continue;

        const deviceClass = device.virtualClass ?? device.class;
        if (![
          'light',
          'socket',
        ].includes(deviceClass)) continue;

        // Create a string of zones, hierarchically
        const zonesArray = [];
        let zone = await device.getZone();
        zonesArray.push(zone);

        while (await zone.getParent() !== null) {
          zone = await zone.getParent();
          zonesArray.push(zone);
        }

        // Create the device object
        // We use numbers instead of device ID to reduce the amount of used tokens
        let deviceIdByNumber = deviceIdsByNumber[device.id];
        if (!deviceIdByNumber) {
          deviceIdByNumber = deviceIdsByNumber[device.id] = Object.keys(deviceIdsByNumber).length + 1;
        }

        result[deviceIdByNumber] = {
          name: device.name,
          type: deviceClass,
          zone: zonesArray.map(zone => zone.name).reverse().join(' -> '),
          state: {},
        };

        if (device.capabilities.includes('onoff')) {
          result[deviceIdByNumber].state.on = device.capabilitiesObj?.onoff?.value;
        }

        if (device.capabilities.includes('dim')) {
          result[deviceIdByNumber].state.brightness = device.capabilitiesObj?.dim?.value * 100;
        }
      }

      return result;
    }

  });
</script>