diff --git a/client.py b/client.py new file mode 100644 index 00000000..c2fd8ace --- /dev/null +++ b/client.py @@ -0,0 +1,75 @@ +import argparse +import asyncio +import logging +import signal + +from aiortc.contrib.signaling import (add_signaling_arguments, + create_signaling) + +from streamclient import StreamClient + +logger = logging.getLogger("pc") + + +async def main(): + parser = argparse.ArgumentParser(description="Data channels ping/pong") + + parser.add_argument( + "--url", type=str, nargs="?", default="http://127.0.0.1:1250/offer" + ) + + parser.add_argument( + "--ping-pong", + help="Benchmark data channel with ping pong", + type=eval, + choices=[True, False], + default="False", + ) + + parser.add_argument( + "--play-from", + type=str, + default="", + ) + add_signaling_arguments(parser) + + args = parser.parse_args() + + signaling = create_signaling(args) + + async def shutdown(signal, loop): + """Cleanup tasks tied to the service's shutdown.""" + logging.info(f"Received exit signal {signal.name}...") + logging.info("Closing database connections") + logging.info("Nacking outstanding messages") + tasks = [t for t in asyncio.all_tasks() if t is not + asyncio.current_task()] + + [task.cancel() for task in tasks] + + logging.info(f"Cancelling {len(tasks)} outstanding tasks") + await asyncio.gather(*tasks, return_exceptions=True) + logging.info(f"Flushing metrics") + loop.stop() + + signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) + loop = asyncio.get_event_loop() + for s in signals: + loop.add_signal_handler( + s, lambda s=s: asyncio.create_task(shutdown(s, loop))) + + # Init client + sc = StreamClient( + signaling=signaling, + url=args.url, + play_from=args.play_from, + ping_pong=args.ping_pong + ) + await sc.start() + print("Stream client started") + async for msg in sc.get_reader(): + print(msg) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/config.ini b/config.ini index 11e4ddcf..c0a41bbf 100644 --- a/config.ini +++ b/config.ini @@ -4,7 +4,7 @@ KMP_DUPLICATE_LIB_OK=TRUE # Export OpenAI API Key OPENAI_APIKEY= # Export Whisper Model Size -WHISPER_MODEL_SIZE=medium +WHISPER_MODEL_SIZE=tiny WHISPER_REAL_TIME_MODEL_SIZE=tiny # AWS config AWS_ACCESS_KEY=***REMOVED*** @@ -18,4 +18,5 @@ BEAM_SIZE=6 MAX_CHUNK_LENGTH=1024 SUMMARIZE_USING_CHUNKS=YES # Audio device -BLACKHOLE_INPUT_AGGREGATOR_DEVICE_NAME=aggregator \ No newline at end of file +BLACKHOLE_INPUT_AGGREGATOR_DEVICE_NAME=aggregator +AV_FOUNDATION_DEVICE_ID=2 \ No newline at end of file diff --git a/Viz-experiments.ipynb b/notebooks/Viz-experiments.ipynb similarity index 100% rename from Viz-experiments.ipynb rename to notebooks/Viz-experiments.ipynb diff --git a/real_time_transcript_with_timestamp_06-21-2023_153233.txt b/real_time_transcript_with_timestamp_06-21-2023_153233.txt deleted file mode 100644 index 1866a60b..00000000 --- a/real_time_transcript_with_timestamp_06-21-2023_153233.txt +++ /dev/null @@ -1 +0,0 @@ -{'text': " that's because I'm looking. I just started the recording so we've got the Python, the WISJACS live real time happening and I can share my screen there and do so we have a letter context but then JDC you can all. Okay, great. That's working. And then, um, they see, you said you can't record the meeting. To start recording. Yeah, I'm going to close my room. I don't have the record about this. we have the real time transcription because Google needs that for the audio plus the transcript plus the timestamps. So cool. Okay, so we can have our discussion as well. I'm planning to begin ask questions or how do you want to go about this? Yeah, so let me make a quick introduction. So what I, it all started with the demo from, uh, uh, Polyantier. So as you know, in the video, so my guess by looking into the videos or I was amazed how much they could achieve just with using like open source simple models. So I still have a lot of experience. start to, it means that they have a definition of entities and actions that they can take. So then they control this to an LLM. So I started exploring with that with Jammo trying to I'm a bit kind of work. So in the NID side, that one good pattern or one good way to achieve this was using GraphQL. So GraphQL is this query language that describes operation mutations and queries. over data and know how to operate. So that means that we shouldn't need to find tune on this. So it's a similar situation of how it works for SQL queries. So people from OpenAI, I am many companies are I mean, that we could map to this, it's Cp because it's some actions that you need to take and you want to out of our human language query, you want to know the parameters of a function to call and which function to. I can take you over and it's amazing that Corey, Meahaw and Chonar at the NIMIKOS. I'm reaching to the point that I would love to have more ideas. So I started by like mocking this kind of entities that could work with ZB. So we have like employees, candidates. I just took a look at out of the comments that we support right now. Reminders. Let me see maybe I'm just sharing One another whole screen. Okay, I think this will solve it So let me go back so you know Cp it's this like common line, we will use it. So there are some entities that are employees. I just like mocked some of them. There is a concept of candidate for the interviews. There is also like reminders. And there are some queries that you can take. So let's say the get-in-play. this is part of the first test that I did. So let me show you here how this can work. So let's say in the first day of the library memory to call how to be. into, sorry, I was in the middle of some, some, eights here. Ten more gods. They always show up. Yeah, yeah. I'm, I'm refructuring like, have a little bit of these, but you see. kong ang to call it. So it works very well for a bunch of cases, you know. So it can map like many queries. So let's say vacations. Thanks. Please. Well, you see, if you... So you see, well, it's very, it's very good for this kind of thing. So it creates the occasion. It feels all the things here. So this can basically be adapted to any like. like, sick cases, such as the one of automating a CLI. So that means adding a common line interface for many things, as long as you can describe the things like this. And this is a very like, like, flexing. So that was the first thing I did, but this has a challenge. What's the challenge that I need to fit this into the LLM? So LLM's have a context window, so you cannot put more text data than it's a program too. the range, let's say word tokens. You know, so if we want to use this for bigger things, we need to come up with an strategy. So that's what I've been doing. So what I did is This I create like a sub tree because this this thing generates us a tree what I mean by this So let's say I know this is a mutation that I need to to to use so there are two things reference here to to to type There is a, I come to employee here and they're not further like references. So, this case is very simple, but there are some others that are there are more complex. So, I've been implementing strategies for this. So, one strategy. So I can use that to create embeddings and then identify or just use that strategy of the top k to know which one should I should include. create the strategy where I directly ask the LLM. So instead of using embeddings, I ask the LLM as if they were tools that is the thing that is now like very popular. So I ask the LLM. This is the query I have. This is the tools that I have to accomplish that with their own. on the screen. I create again this, this like sub tree of the graph QL and then I can obtain the query. So that was a challenge because even that strategy with using the other lem, in the case of of says me, concretely that. It's number of queries is so big. Unutations, that it doesn't fit. So I need to come up with the embedding strategy. So there, you usually, I mean, you can select strategies. So let's say, So this is the query I'm giving for the Rails organization get their Rails repo on the last full request with status open. This is the graphQL that I'm passing. The schema strategy I'm using embeddings. So I'm just creating them bedding. and Did it in our way and stable state right now? Let me try to uncommon some things that I'm I Of course. Yeah, it's very funny to use the LOM strategy here. So did you actually can take a while? and just getting the parts is too big but with the anthropic LLM I can make it it does fit. So, how many tokens is there if I mean like... How many tokens is there? But to be able to fit that to the LLM, you need an strategy to tell the LLM which parts it needs to use. So there are two strategies that I implemented. One is using embeddings. So I grab the with. book and answer this. So I cannot definitely fit a GitHub GraphQL because look how big it is, not even in the in the Vigger ones, in the Vigger and Tropic. So it's 50. So even the 100k window of this LLM. So I just need to pick the entities that are relevant with whatever strategy. Right now there are two like LLMs and embeddings. So I managed to get it that thing queries are they all, you know, do they have rich comments like that? Like most of of big APIs do have that. Okay. And that's a requirement. And because if you got if you got rid of the comments than the LM would not maybe note as well. Yeah, as well. But like I guess there's a trade off rate like maybe you could eliminate the comments and get a better result. because not like foretell that. So I've implemented some like strategies. So for instance, I have one where I can just fit in the like the tools that I have available. And then when I include the graph goes reasons but because it just doesn't fit but yeah all of those combinations are available I can throw a control in a graph QL without comments and it might work if if this variable names are self descriptive as you know as you have seen these one of these things work. So there is this explorer from see. So I ask this get They're rosfick. Just by. And it's just really complex. That's the output that you got from your script right there. And this is the output. These one on the left hand side or the one that is highlighted. that's what I got and then he got the stars and then you can. So that's let me try another one. Yeah, and maybe I could share a use case that where we were talking about, okay, see. building. They're really pushing tons of small meetings as I mean, if I just deliverage these open data standards to collect data for like incompatible appliances, like let's say Mixers on a manufacturing line from different manufacturing. actors to do that with just their knowledge. So creating the open data standards is a chunk of the work. Then improving the gooey interface that they have to then create these graph QL queries is another and I think where We can specific appliances and data elements. How do we take that as that further, basically use language or jdc's describing here to say, hey, I want to get a historian graph for all of the mixers that we had on the line. And that would put that specific data for them, right? That's kind of the idea we're going to explore at least. So let me show you, I already implemented SESME. So let me show you this. So what I did, I went to describing. So they have this query that it's not an easy one. So return a list of times here is sample values for a given instance, that we'll specify time range. So I did, I further described this. because it's within the start time blah blah blah and time. Take a look at this query and it's exactly just a couple of more things just the first. But it's basically got it right. So get raw data with. I don't know if necessarily the users of this would be very clear about the information they need and have an awareness of that. I understand most engineers in manufacturing spaces are familiar with that type of notation. but I do say they select or I mean it's a ghost thing but let's say I would say I meet night and I think this is a new. But I think if you manage to squeeze a Yeah, okay. There we go. Well, it's very fast, too. That's yeah, because I'm using on tropic and I'm managed to use a cheaper model. That's another story of like tweaking the prompt. So I could like first. This is quick, but right now it's like the embellings. It's using like embelling from OpenAI and getting the tools that I need to use. And then when I get the graph QL, I use it in Anthropic. 15,000 yeah yeah 15,000 just let's say like 30k 30k tokens making our shift so that goes into every I guess prompt has to have all 30,000 I don't get the quick. What I do too. I can give as much as 100K tokens as context that it's the prom blues all the day that that include. And just the parts that are really one to the query. And we've done some research. Yeah, and I guess my question is if you are passing the full 15,000 lines with every query or every But it's less than that because the whole idea, I mean, once let me, let me explain you better. So the strategy that it's the embeddings was able to identify that I need to use this, these two fields. I see. Okay. and the So, this query further references this filter, that includes a bunch of other fields. You get the point like it's these things like reference other types so you need to include them like all Yeah, yeah, you do like an LLM kind of tree shaking To minimize yeah, yeah, so first I identify which mutations or queries I will need And I'll include just the things that I will know. I will do the same thing. And all the dependencies. Yeah, exactly. Yeah. I'm doing this. Yeah. I was wondering what do you embed exactly? Is it only the human language? which this is the the the this is the graphQL prompt that I use for anthropic so it says I will give you the following graphQL schema and I include the this smaller schema that I'm able to. figure and it's able to figure it out. Like these rules, the context they time and that's it. Yeah, but on the open AI site, when you generate the embeddings to to find chemist strategies just like an interface. So full schema, whatever schema I'm pass I return the whole schema. LLM strategy. So it it works like this. Get two descriptions by operation. So what it does. guys, let's say I get this, just this part, let's just code of conduct and then I put two dots and say like this, look up the code of conduct by ski. So I pass all of this in this format. You see here that key value? That's the LLM strategy. Does it incorporate the comment on that type name as well? Because it's like the whole tree right? Yeah. And this gets plugged into these tools from your health laging that they're me al-Qoma. So that's the LLM strategy. The the other strategy is simpler and it's embedding strategy. So for the embedding strategy I basically construct these document. the LLM strategy do you run the LLM once for every single field of every type in the source schema? No, no, no, it's not I chunk I I chunk this like code of condo to dot look at times that can like be even bigger than the context window, but I'm not talking that much, but that's not like very hard to do. It's just like have a buffer, yeah. Yeah, margin ever. Yeah. And limitations and it still it works. It doesn't work for Sesame so that's why I wrote embeddings Because embeddings, you know, I just create embedding make the query and Come up with the tools. Let's say the tools are the queries and mutations that I will need get queries, a mutation, descriptions, and then I create this document using the jamaindex capability. So here is basically the same I just described, let's say it will say like enterprise and look up and enterprise, get keys, and then I return the the proper documents. So this is what it identifies the operations ordered by a score and then build the schema from operation. That's what I call the thing. Once I know But remarkably, it's able to tackle these very huge schemas really well. You see, it's got its like... JDC, what are yours? So two questions. One, I have a question. and see what you thought on that. Right now, there is a repo for it. I think it's usable already there. What I'm trying to do now is just clean it up because I have this Frankenstein. that UL, I could use like hydropen AI or antropic. I do need to use antropic for the bigger ones. So if you just want to play with the CP, there won't be any trouble. So it's here. This schema, I know the query that generates, I can Let me show you this Yeah, I made just wait until you're done with your refact going to do the test. Awesome examples and maybe like two or three examples at like our totally out of scope and not feasible and then take some time to demo that to them and be set the conversation and hate like it's like how much this is there are like, you know, they're afraid of using a graph to algorithm it to structure endpoints and then dump them into some sort of a data explorer, right? Like if we were to create a webinar phase that presented the input, But for example, if we put some sort of like a graph graph on a type thing, whatever, on the third and third end there, and wondering like how much of this that how elegant that could be really, right? And it doesn't have to be perfect, necessarily, but at least a way that it can like infer the date that they are using here if they are just at least of all attributes and it has worked like pretty well. So query equipment, you see these one equipment. Yeah. I queries that already prepared that then they put into some dashboard or anything Because in the end you will need, I think you will need some expertise because sometimes it's unless you describe all the fields it will miss some fields or I think the way that this is currently used is when people want to create specific reports and specific interfaces for, again, key questions. Like give me this particular data for the data. I think that that's really how our customers are utilizing them to create these kind of custom reports. But like what if that didn't have to happen? And instead you could have that natural language query that maybe works for like the simpler or majority of requests. I'm sorry. Good for you since it's a graphana visualization of that, right? I instruct you like query places for this play name. So it did well. It got this play, but for any reason, here with that graphQL. So it says cannot query field places of type query. Did you mean place? You know the solution is places. So I don't know if there is some inconsistency in the graphQL, but that's another story like it got it that's like a post-process, for these very specific errors, right? Basically, it was all of them. Yeah, yeah. That's... I kind of see like more potential on some other... There are some efforts on it, so that's also worth mentioning maybe to the guys here. So, these people from Stanford tried to make basically this kind of thing. of the sales have the context window that it's your restriction. So what these guys did, they took a cheaper LLM and that is some Yamaha and fine-tune it on the instructions. I think because they tried to format everything as function calls and I think this thing has seen more or some function called, you know, there is this coma and things. GraphQL, it's kind of simpler. You know, just know the field. You might miss some fields and still the things that goes on. So it's kind of in between in the spectrum of of things like rest, but in the other end you have Jason, free running Jason and in between you have like rest and graphQL. So that's kind of the reasoning I also used to. to be a functionality to call to do this kind of thing. I think behind the scenes they are doing the same. So it doesn't one can just use the same thing and it will take advantage of the training that that they, yeah, okay. Yeah, and I know it would be great. Sorry. Sorry, go ahead. I was just going to say that I still really do passionately feel about talking to think I Q about. So after you refactor, I'll take it. I'll run with I'll back there. We could get some under payouts to build this and we could learn. So I think there's potential here. I really want to give it a shot. At least throw it at them and get there perspective and then you know, I can summarize those learnings for us to think about it. I'll run it on my end and try to copy some demos. I will say that you can just do that straight. Just let me know when I will put it into our working state with the. I'm just gonna put you all in next week. There is some... It says that it has an error, but it's not like a run-use. That's something I don't understand sometimes. GraphQL. Oh, instruct it. off and so on close out that I Yeah, so you can see here Rob Octifrecord also a response to the question of issues. So qualify as a situation. It's here. It's in the get rid of a 33-jump. You see it's. and it has inams and things like that are not like trivial. This is an inam. This is this kind of nesting pattern that it needs to use. So I feel very confident with the the thing, some more ideas. One idea is that this can form the basis of some assistant, local assistant, so you could instruct like, hey, open the file, browse to the window, like a smarter, that's It's an overlay with the other thing that we were speaking. I know if you remember, Sean, like this, how do you call that these URLs internal URLs? These people are moving in that direction. So it's basically the same thing like you see like they're proposing as martyr-barrel. You can basically do it's the same thing. Yeah, okay. So this is the most general use case like this text interface for many things. I will also recommend you guys to see this, this is the one. And this one is even easier because you know what was the latest like entity involved. So you can hint the model to use that like subgraph let's say. So Sananti team will there. So you can just like stuff the prompt like show me more details, but they are referencing these entities. So you know immediately what's the query that it's involved there and And it's remarkable that these guys, there is some place where it's shown that they are using open source model here. You see they disclose this. So there's a slide. Yeah. GPD or maybe at the start of ontology prompt and I have I was analyzing all of all of these screens that they disclose. Yeah. Remember the They also, they don't have this problem of what it can access or not. It's just like another user. So whatever that user has access to the LLM has access to. And this is the actions that it can take. They include one thing. called Slysofau. They so instead of getting this PII problem and adding to the model, they just put an input filter to whatever PII it's involved. They just got it. Then the model and then there is the valuation. So this is like a whole article. Keep that straight. You want to about the ask a question earlier? Uh, yes. So for the output of the GraphQL, are you using the, like, strange generation or whatever, or is it just opening this syntactically correct GraphQL like on its own? concrete schema. Now nowadays you can do that but I don't know if you can describe GraphQL. You can describe a JSON schema at least in OpenAI. Yeah, but valid queries for that schema, like can you force to open that query experience even without respect to the schema? Sorry, I kind of. we could just maybe use constraint generation to make sure that at least it opens. It's actually great. We're happy to out even if it doesn't know about the schema you know, but it would be even better to use the constraint generation to with knowledge of the schema. Yeah, definitely. I do things like let me show you I think using main I have a drag it's So I first heard Rory is that it doesn't speed like valid. GraphQL are all, but I've never encountered that. Maybe sometimes it's missing like a mutation or query, but the prompts that I'm using are already like covering for that. but in this format, something like that. So for that matter, I use a reggae, a general reggae for refuel queries or mutations. Oh, like an extract part of the result. Yeah, and then I do the validation. in that front. But yeah, you could like constrain. I think there are some efforts and I dare I read a debate of what was open AI really doing because because you can like feed it back and tell it hey, this is bad, generate again, that's one solution or maybe run it with many temperatures and select the valid ones, that's another solution that I've seen. And finally, in my ear and then you can like squash the the probability because each like in that layer you will have one one output per tokens so there will be some invalid tokens so you can fill some guy that did that okay. It's trained like only on valid graph QL maybe that would reduce the full capsize. You know what I mean? Yeah, yeah, that's that's kind of the of the of the approach. that we are having to select the tool. It's just the training that speeds that. Yeah, they find the model just for the but it has some like small errors. I mean I think I prefer to have a more way more powerful LLM rather than have that from what I've seen it's very easy just to use the the reggae, the function calls are like they kind of intuitive when you read them at a way, right? So yeah, I think like a foundation of model, it's like really helps in that situation. Yeah, I've seen that these models, we can look English to French and it's selected the French model. You see? So there should be, I don't know why I think this is an example in their showcase that they are being honest. It just used the model. If you go to the data, they mostly use French, so that's completely biased, it's like overfitted. So I kind of prefer the more general model than trying to, I don't know it's. these but for SQL and if you read the paper like very with a lot of attention they didn't gain much by fine tuning. You can find it here. fine tuning maybe. So Google train an SQL model, this palm, they are comparing it with others. So look the figures. The difference between fine-tune and and a few shots, 60, 7.4, bound with a few shot, so 77.3 to 78.3, 77.3 to 7.3, look. the stuff model to a fine tune model. It was just 1% so I really prefer to have the few shots. That's something that it's also emerging. So once you get the query, you can use also embeddings to get the most of this data is really. It's parts, but I do prefer to use that. And if you see, Langchain, Langchain, let me get Langchain Docs Langchain. Well, they change their docs, but let me show you some examples. or retreat. Well, they have this concept, you know, that on the basis of the prompt, they fetch the most relevant. Yeah, by that some point in my make sense to fine tune. I'm looking forward to bringing on my machine and kind of understanding a little bit deeper. And yeah, the exploring more use cases at the sounds really exciting as well. So this is basically for CPIT, it works really well because it's small and everything can be Is there a way of this and attempted to deploy this to help with SIP and all is that a feasible thing? I mean yes, because it's not private we can't really do that. Well at this point, just to comply with monadico. That's where Sean's ideas about how to like overcome the context when those sides and Local processing would avoid Maybe help if it's if it was female that's it sir. No, it's just using like like a privately host I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, information, right? We actually have a pre-processor to resolve what we're seeing on the screen right now. But yeah, there's still some struggles in the local model to get for figure out and get creative with. I do wonder if a summarization focuss could be useful for that token. Contact size prop as well. Like if something I could be applied to this scheme, get it fixed. But I'm going to stop recording. Yeah, I got that. And I appreciate everyone taking time of their day to come here and explore this and kind of help explore how to keep driving.", 'chunks': [{'timestamp': (0.0, 8.08), 'text': " that's because I'm looking. I just started the recording so we've got the Python, the WISJACS live real time happening and I can share my screen there and do so we have a letter context but then JDC"}, {'timestamp': (8.08, 15.52), 'text': " you can all. Okay, great. That's working. And then, um, they see, you said you can't record the meeting. To start recording. Yeah, I'm going to close my room. I don't have the record about this."}, {'timestamp': (15.52, 24.52), 'text': ' we have the real time transcription because Google needs that for the audio plus the transcript plus the timestamps. So cool. Okay, so we can have our discussion as well.'}, {'timestamp': (24.52, 27.52), 'text': " I'm planning to begin ask questions or how do you want to go about this? Yeah, so let me make a quick introduction. So what I, it all started with the demo from, uh, uh, Polyantier."}, {'timestamp': (27.52, 39.519999999999996), 'text': ' So as you know, in the video, so my guess by looking into the videos or I was amazed how much they could achieve just with using like open source simple models. So I still have a lot of experience.'}, {'timestamp': (39.519999999999996, 47.12), 'text': ' start to, it means that they have a definition of entities and actions that they can take. So then they control this to an LLM. So I started exploring with that with Jammo trying to'}, {'timestamp': (47.12, 55.12), 'text': " I'm a bit kind of work. So in the NID side, that one good pattern or one good way to achieve this was using GraphQL. So GraphQL is this query language that describes operation mutations and queries."}, {'timestamp': (55.12, 61.839999999999996), 'text': " over data and know how to operate. So that means that we shouldn't need to find tune on this. So it's a similar situation of how it works for SQL queries. So people from OpenAI, I am many companies are"}, {'timestamp': (61.839999999999996, 76.84), 'text': " I mean, that we could map to this, it's Cp because it's some actions that you need to take and you want to out of our human language query, you want to know the parameters of a function to call and which function to."}, {'timestamp': (76.84, 83.84), 'text': " I can take you over and it's amazing that Corey, Meahaw and Chonar at the NIMIKOS. I'm reaching to the point that I would love to have more ideas."}, {'timestamp': (83.84, 91.84), 'text': ' So I started by like mocking this kind of entities that could work with ZB. So we have like employees, candidates. I just took a look at out of the comments that we support right now. Reminders.'}, {'timestamp': (91.84, 93.84), 'text': " Let me see maybe I'm just sharing One another whole screen. Okay, I think this will solve it So let me go back so you know Cp it's this like common"}, {'timestamp': (93.84, 95.60000000000001), 'text': " line, we will use it. So there are some entities that are employees. I just like mocked some of them. There is a concept of candidate for the interviews. There is also like reminders. And there are some queries that you can take. So let's say the get-in-play."}, {'timestamp': (95.60000000000001, 101.88000000000001), 'text': " this is part of the first test that I did. So let me show you here how this can work. So let's say in the first day of the library memory to call how to be."}, {'timestamp': (101.88000000000001, 106.24000000000001), 'text': " into, sorry, I was in the middle of some, some, eights here. Ten more gods. They always show up. Yeah, yeah. I'm, I'm refructuring like, have a little bit of these, but you see."}, {'timestamp': (106.24000000000001, 133.10000000000002), 'text': ' kong ang'}, {'timestamp': (133.10000000000002, 143.18000000000004), 'text': " to call it. So it works very well for a bunch of cases, you know. So it can map like many queries. So let's say vacations."}, {'timestamp': (143.18000000000004, 145.18000000000004), 'text': ' Thanks.'}, {'timestamp': (145.18000000000004, 147.18000000000004), 'text': ' Please. Well, you see, if you...'}, {'timestamp': (147.18000000000004, 153.18000000000004), 'text': " So you see, well, it's very, it's very good for this kind of thing. So it creates the occasion. It feels all the things here. So this can basically be adapted to any like."}, {'timestamp': (153.18000000000004, 157.98000000000005), 'text': ' like, sick cases, such as the one of automating a CLI. So that means adding a common line interface for many things, as long as you can describe the things like this. And this is a very like, like, flexing.'}, {'timestamp': (157.98000000000005, 161.98000000000005), 'text': " So that was the first thing I did, but this has a challenge. What's the challenge that I need to fit this into the LLM? So LLM's have a context window, so you cannot put more text data than it's a program too."}, {'timestamp': (161.98000000000005, 164.98000000000005), 'text': " the range, let's say word tokens. You know, so if we want to use this for bigger things, we need to come up with an strategy. So that's what I've been doing. So what I did is"}, {'timestamp': (164.98000000000005, 170.92000000000004), 'text': " This I create like a sub tree because this this thing generates us a tree what I mean by this So let's say I know this is a mutation that I need to to to use so there are two things reference here to to to type"}, {'timestamp': (170.92000000000004, 181.92000000000004), 'text': " There is a, I come to employee here and they're not further like references. So, this case is very simple, but there are some others that are there are more complex. So, I've been implementing strategies for this. So, one strategy."}, {'timestamp': (181.92000000000004, 196.92000000000004), 'text': ' So I can use that to create embeddings and then identify or just use that strategy of the top k to know which one should I should include.'}, {'timestamp': (196.92000000000004, 201.68000000000004), 'text': ' create the strategy where I directly ask the LLM. So instead of using embeddings, I ask the LLM as if they were tools that is the thing that is now like very popular. So I ask the LLM. This is the query I have. This is the tools that I have to accomplish that with their own.'}, {'timestamp': (201.68000000000004, 207.68000000000004), 'text': ' on the screen. I create again this, this like sub tree of the graph QL and then I can obtain the query. So that was a challenge because even that strategy with using the other lem, in the case of of says me, concretely that.'}, {'timestamp': (207.68000000000004, 210.64000000000004), 'text': " It's number of queries is so big. Unutations, that it doesn't fit. So I need to come up with the embedding strategy. So there, you usually, I mean, you can select strategies. So let's say,"}, {'timestamp': (210.64000000000004, 218.64000000000004), 'text': " So this is the query I'm giving for the Rails organization get their Rails repo on the last full request with status open. This is the graphQL that I'm passing. The schema strategy I'm using embeddings. So I'm just creating them bedding."}, {'timestamp': (218.64000000000004, 220.64000000000004), 'text': " and Did it in our way and stable state right now? Let me try to uncommon some things that I'm I"}, {'timestamp': (220.64000000000004, 222.64000000000004), 'text': " Of course. Yeah, it's very funny to use the LOM strategy here. So did you actually can take a while?"}, {'timestamp': (222.64000000000004, 229.64000000000004), 'text': ' and just getting the parts is too big but with the anthropic LLM I can make it it does fit. So, how many tokens is there if I mean like... How many tokens is there?'}, {'timestamp': (229.64000000000004, 237.64000000000004), 'text': ' But to be able to fit that to the LLM, you need an strategy to tell the LLM which parts it needs to use. So there are two strategies that I implemented. One is using embeddings. So I grab the with.'}, {'timestamp': (237.64000000000004, 246.20000000000005), 'text': " book and answer this. So I cannot definitely fit a GitHub GraphQL because look how big it is, not even in the in the Vigger ones, in the Vigger and Tropic. So it's 50."}, {'timestamp': (246.20000000000005, 249.56000000000006), 'text': ' So even the 100k window of this LLM. So I just need to pick the entities that are relevant with whatever strategy. Right now there are two like LLMs and embeddings. So I managed to get it that thing'}, {'timestamp': (249.56000000000006, 255.56000000000006), 'text': " queries are they all, you know, do they have rich comments like that? Like most of of big APIs do have that. Okay. And that's a requirement."}, {'timestamp': (255.56000000000006, 262.2800000000001), 'text': " And because if you got if you got rid of the comments than the LM would not maybe note as well. Yeah, as well. But like I guess there's a trade off rate like maybe you could eliminate the comments and get a better result."}, {'timestamp': (262.2800000000001, 269.0800000000001), 'text': " because not like foretell that. So I've implemented some like strategies. So for instance, I have one where I can just fit in the like the tools that I have available. And then when I include"}, {'timestamp': (269.0800000000001, 274.2800000000001), 'text': " the graph goes reasons but because it just doesn't fit but yeah all of those combinations are available I can throw a control in a graph QL without comments and it might work if if this variable names are self descriptive as you know as you have seen these"}, {'timestamp': (274.2800000000001, 284.2800000000001), 'text': ' one of these things work. So there is this explorer from see. So I ask this get'}, {'timestamp': (284.2800000000001, 286.2800000000001), 'text': " They're rosfick. Just by. And it's just really complex. That's the output that you got from your script right there. And this is the output. These one on the left hand side or the one that is highlighted."}, {'timestamp': (286.2800000000001, 292.2800000000001), 'text': " that's what I got and then he got the stars and then you can. So that's let me try another one. Yeah, and maybe I could share a use case that where we were talking about, okay, see."}, {'timestamp': (292.2800000000001, 296.68000000000006), 'text': " building. They're really pushing tons of small meetings as I mean, if I just deliverage these open data standards to collect data for like incompatible appliances, like let's say Mixers on a manufacturing line from different manufacturing."}, {'timestamp': (296.68000000000006, 303.68000000000006), 'text': ' actors to do that with just their knowledge. So creating the open data standards is a chunk of the work. Then improving the gooey interface that they have to then create these graph QL queries is another and I think where'}, {'timestamp': (303.68000000000006, 306.68000000000006), 'text': " We can specific appliances and data elements. How do we take that as that further, basically use language or jdc's describing here to say, hey, I want to get a historian graph for all of the mixers that we had on the line."}, {'timestamp': (306.68000000000006, 311.68000000000006), 'text': " And that would put that specific data for them, right? That's kind of the idea we're going to explore at least. So let me show you, I already implemented SESME. So let me show you this. So what I did,"}, {'timestamp': (311.68000000000006, 316.9000000000001), 'text': " I went to describing. So they have this query that it's not an easy one. So return a list of times here is sample values for a given instance, that we'll specify time range. So I did, I further described this."}, {'timestamp': (316.9000000000001, 320.5000000000001), 'text': " because it's within the start time blah blah blah and time. Take a look at this query and it's exactly just a couple of more things just the first. But it's basically got it right. So get raw data with."}, {'timestamp': (320.5000000000001, 327.5000000000001), 'text': " I don't know if necessarily the users of this would be very clear about the information they need and have an awareness of that. I understand most engineers in manufacturing spaces are familiar with that type of notation."}, {'timestamp': (327.5000000000001, 333.8200000000001), 'text': " but I do say they select or I mean it's a ghost thing but let's say I would say I meet night and I think this is a new. But I think if you manage to squeeze a"}, {'timestamp': (333.8200000000001, 348.8200000000001), 'text': " Yeah, okay. There we go. Well, it's very fast, too. That's yeah, because I'm using on tropic and I'm managed to use a cheaper model. That's another story of like tweaking the prompt. So I could like first."}, {'timestamp': (348.8200000000001, 352.0600000000001), 'text': " This is quick, but right now it's like the embellings. It's using like embelling from OpenAI and getting the tools that I need to use. And then when I get the graph QL, I use it in Anthropic."}, {'timestamp': (352.0600000000001, 362.22000000000014), 'text': " 15,000 yeah yeah 15,000 just let's say like 30k 30k tokens making our shift so that goes into every I guess prompt has to have all 30,000"}, {'timestamp': (362.22000000000014, 364.22000000000014), 'text': " I don't get the quick. What I do too. I can give as much as 100K tokens as context that it's the prom blues all the day that that include."}, {'timestamp': (364.22000000000014, 368.22000000000014), 'text': " And just the parts that are really one to the query. And we've done some research. Yeah, and I guess my question is if you are passing the full 15,000 lines with every query or every"}, {'timestamp': (368.22000000000014, 375.22000000000014), 'text': " But it's less than that because the whole idea, I mean, once let me, let me explain you better. So the strategy that it's the embeddings was able to identify that I need to use this, these two fields. I see. Okay."}, {'timestamp': (375.22000000000014, 382.22000000000014), 'text': ' and the'}, {'timestamp': (382.22000000000014, 394.22000000000014), 'text': ' So, this query further references this filter, that includes a bunch of other fields.'}, {'timestamp': (394.22000000000014, 400.22000000000014), 'text': " You get the point like it's these things like reference other types so you need to include them like all Yeah, yeah, you do like an LLM kind of tree shaking To minimize yeah, yeah, so first I identify which mutations or queries I will need"}, {'timestamp': (400.22000000000014, 403.22000000000014), 'text': " And I'll include just the things that I will know. I will do the same thing. And all the dependencies. Yeah, exactly. Yeah. I'm doing this. Yeah. I was wondering what do you embed exactly? Is it only the human language?"}, {'timestamp': (403.22000000000014, 418.02000000000015), 'text': " which this is the the the this is the graphQL prompt that I use for anthropic so it says I will give you the following graphQL schema and I include the this smaller schema that I'm able to."}, {'timestamp': (418.02000000000015, 426.02000000000015), 'text': " figure and it's able to figure it out. Like these rules, the context they time and that's it. Yeah, but on the open AI site, when you generate the embeddings to to"}, {'timestamp': (426.02000000000015, 431.70000000000016), 'text': " find chemist strategies just like an interface. So full schema, whatever schema I'm pass I return the whole schema. LLM strategy. So it it works like this. Get two descriptions by operation. So what it does."}, {'timestamp': (431.70000000000016, 434.94000000000017), 'text': " guys, let's say I get this, just this part, let's just code of conduct and then I put two dots and say like this, look up the code of conduct by ski. So I pass all of this in this format. You see here that key value?"}, {'timestamp': (434.94000000000017, 436.94000000000017), 'text': " That's the LLM strategy. Does it incorporate the comment on that type name as well? Because it's like the whole tree right? Yeah. And this gets plugged into these tools from your health laging that they're"}, {'timestamp': (436.94000000000017, 446.14000000000016), 'text': " me al-Qoma. So that's the LLM strategy. The the other strategy is simpler and it's embedding strategy. So for the embedding strategy I basically construct these document."}, {'timestamp': (446.14000000000016, 453.14000000000016), 'text': " the LLM strategy do you run the LLM once for every single field of every type in the source schema? No, no, no, it's not I chunk I I chunk this like code of condo to dot"}, {'timestamp': (453.14000000000016, 458.1800000000002), 'text': " look at times that can like be even bigger than the context window, but I'm not talking that much, but that's not like very hard to do. It's just like have a buffer, yeah. Yeah, margin ever. Yeah. And"}, {'timestamp': (458.1800000000002, 462.94000000000017), 'text': " limitations and it still it works. It doesn't work for Sesame so that's why I wrote embeddings Because embeddings, you know, I just create embedding make the query and Come up with the tools. Let's say the tools are the queries and mutations that I will need"}, {'timestamp': (462.94000000000017, 470.94000000000017), 'text': " get queries, a mutation, descriptions, and then I create this document using the jamaindex capability. So here is basically the same I just described, let's say it will say like enterprise and look up and"}, {'timestamp': (470.94000000000017, 477.94000000000017), 'text': " enterprise, get keys, and then I return the the proper documents. So this is what it identifies the operations ordered by a score and then build the schema from operation. That's what I call the thing. Once I know"}, {'timestamp': (477.94000000000017, 484.94000000000017), 'text': " But remarkably, it's able to tackle these very huge schemas really well. You see, it's got its like... JDC, what are yours? So two questions. One, I have a question."}, {'timestamp': (484.94000000000017, 487.94000000000017), 'text': " and see what you thought on that. Right now, there is a repo for it. I think it's usable already there. What I'm trying to do now is just clean it up because I have this Frankenstein."}, {'timestamp': (487.94000000000017, 494.90000000000015), 'text': " that UL, I could use like hydropen AI or antropic. I do need to use antropic for the bigger ones. So if you just want to play with the CP, there won't be any trouble. So it's here."}, {'timestamp': (494.90000000000015, 501.90000000000015), 'text': " This schema, I know the query that generates, I can Let me show you this Yeah, I made just wait until you're done with your refact going to do"}, {'timestamp': (501.90000000000015, 507.44000000000017), 'text': " the test. Awesome examples and maybe like two or three examples at like our totally out of scope and not feasible and then take some time to demo that to them and be set the conversation and hate like it's like"}, {'timestamp': (507.44000000000017, 509.26000000000016), 'text': " how much this is there are like, you know, they're afraid of using a graph to algorithm it to structure endpoints and then dump them into some sort of a data explorer, right? Like if we were to create a webinar phase that presented the input,"}, {'timestamp': (509.26000000000016, 513.9000000000002), 'text': " But for example, if we put some sort of like a graph graph on a type thing, whatever, on the third and third end there, and wondering like how much of this that how elegant that could be really, right? And it doesn't have to be perfect, necessarily, but at least"}, {'timestamp': (513.9000000000002, 518.4200000000002), 'text': ' a way that it can like infer the date that they are using here if they are just at least of all attributes and it has worked like pretty well. So query equipment, you see these one equipment. Yeah.'}, {'timestamp': (518.4200000000002, 523.4200000000002), 'text': " I queries that already prepared that then they put into some dashboard or anything Because in the end you will need, I think you will need some expertise because sometimes it's unless you describe all the fields it will miss some fields or"}, {'timestamp': (523.4200000000002, 530.4200000000002), 'text': ' I think the way that this is currently used is when people want to create specific reports and specific interfaces for, again, key questions. Like give me this particular data for the data.'}, {'timestamp': (530.4200000000002, 536.4200000000002), 'text': " I think that that's really how our customers are utilizing them to create these kind of custom reports. But like what if that didn't have to happen? And instead you could have that natural language query that maybe works for like the simpler or majority of requests."}, {'timestamp': (536.4200000000002, 538.4200000000002), 'text': " I'm sorry. Good for you since it's a graphana visualization of that, right? I instruct you like query places for this play name. So it did well. It got this play, but for any reason,"}, {'timestamp': (538.4200000000002, 543.9400000000002), 'text': " here with that graphQL. So it says cannot query field places of type query. Did you mean place? You know the solution is places. So I don't know if there is some inconsistency in the graphQL, but that's another story like it got it that's"}, {'timestamp': (543.9400000000002, 547.9400000000002), 'text': " like a post-process, for these very specific errors, right? Basically, it was all of them. Yeah, yeah. That's... I kind of see like more potential on some other..."}, {'timestamp': (547.9400000000002, 554.9400000000002), 'text': " There are some efforts on it, so that's also worth mentioning maybe to the guys here. So, these people from Stanford tried to make basically this kind of thing."}, {'timestamp': (554.9400000000002, 561.5800000000002), 'text': " of the sales have the context window that it's your restriction. So what these guys did, they took a cheaper LLM and that is some Yamaha and fine-tune it on the instructions."}, {'timestamp': (561.5800000000002, 576.5800000000002), 'text': ' I think because they tried to format everything as function calls and I think this thing has seen more'}, {'timestamp': (576.5800000000002, 579.9400000000002), 'text': " or some function called, you know, there is this coma and things. GraphQL, it's kind of simpler. You know, just know the field. You might miss some fields and still the things that goes on. So it's kind of in between in the spectrum of"}, {'timestamp': (579.9400000000002, 588.6400000000002), 'text': " of things like rest, but in the other end you have Jason, free running Jason and in between you have like rest and graphQL. So that's kind of the reasoning I also used to."}, {'timestamp': (588.6400000000002, 593.4000000000002), 'text': " to be a functionality to call to do this kind of thing. I think behind the scenes they are doing the same. So it doesn't one can just use the same thing and it will take advantage of the training that"}, {'timestamp': (593.4000000000002, 605.4000000000002), 'text': " that they, yeah, okay. Yeah, and I know it would be great. Sorry. Sorry, go ahead. I was just going to say that I still really do passionately feel about talking to think I Q about. So after you refactor, I'll take it. I'll run with"}, {'timestamp': (605.4000000000002, 609.4000000000002), 'text': " I'll back there. We could get some under payouts to build this and we could learn. So I think there's potential here. I really want to give it a shot. At least throw it at them and get there perspective and then you know, I can summarize those learnings for us to think about it."}, {'timestamp': (609.4000000000002, 612.4000000000002), 'text': " I'll run it on my end and try to copy some demos. I will say that you can just do that straight. Just let me know when I will put it into our working state with the."}, {'timestamp': (612.4000000000002, 614.4000000000002), 'text': " I'm just gonna put you all in next week. There is some... It says that it has an error, but it's not like a run-use. That's something I don't understand sometimes. GraphQL. Oh, instruct it."}, {'timestamp': (614.4000000000002, 616.4000000000002), 'text': ' off and so on close out that I Yeah, so you can see here Rob Octifrecord'}, {'timestamp': (616.4000000000002, 622.4000000000002), 'text': " also a response to the question of issues. So qualify as a situation. It's here. It's in the get rid of a 33-jump. You see it's."}, {'timestamp': (622.4000000000002, 627.0000000000002), 'text': ' and it has inams and things like that are not like trivial. This is an inam. This is this kind of nesting pattern that it needs to use. So I feel very confident with the'}, {'timestamp': (627.0000000000002, 634.6000000000003), 'text': " the thing, some more ideas. One idea is that this can form the basis of some assistant, local assistant, so you could instruct like, hey, open the file, browse to the window, like a smarter, that's"}, {'timestamp': (634.6000000000003, 637.2800000000002), 'text': " It's an overlay with the other thing that we were speaking. I know if you remember, Sean, like this, how do you call that these URLs internal URLs? These people are moving in that direction."}, {'timestamp': (637.2800000000002, 643.8000000000002), 'text': " So it's basically the same thing like you see like they're proposing as martyr-barrel. You can basically do it's the same thing. Yeah, okay."}, {'timestamp': (643.8000000000002, 654.8000000000002), 'text': ' So this is the most general use case like this text interface for many things. I will also recommend you guys to see this, this is the one.'}, {'timestamp': (654.8000000000002, 661.8000000000002), 'text': " And this one is even easier because you know what was the latest like entity involved. So you can hint the model to use that like subgraph let's say."}, {'timestamp': (661.8000000000002, 677.8000000000002), 'text': " So Sananti team will there. So you can just like stuff the prompt like show me more details, but they are referencing these entities. So you know immediately what's the query that it's involved there and"}, {'timestamp': (677.8000000000002, 688.8400000000001), 'text': " And it's remarkable that these guys, there is some place where it's shown that they are using open source model here. You see they disclose this. So there's a slide. Yeah."}, {'timestamp': (688.8400000000001, 695.5600000000002), 'text': ' GPD or maybe at the start of ontology prompt and I have I was analyzing all of all of these screens that they disclose. Yeah. Remember the'}, {'timestamp': (695.5600000000002, 701.3200000000002), 'text': " They also, they don't have this problem of what it can access or not. It's just like another user. So whatever that user has access to the LLM has access to. And this is the actions that it can take. They include one thing."}, {'timestamp': (701.3200000000002, 716.3200000000002), 'text': " called Slysofau. They so instead of getting this PII problem and adding to the model, they just put an input filter to whatever PII it's involved. They just got it. Then the model and then there is the valuation. So this is like a whole article."}, {'timestamp': (716.3200000000002, 717.3200000000002), 'text': ' Keep that straight. You want to about the ask a question earlier? Uh, yes. So for the output of the GraphQL, are you using the, like, strange generation or whatever, or is it just opening this syntactically correct GraphQL like on its own?'}, {'timestamp': (717.3200000000002, 723.3200000000002), 'text': " concrete schema. Now nowadays you can do that but I don't know if you can describe GraphQL. You can describe a JSON schema at least in OpenAI."}, {'timestamp': (723.3200000000002, 735.1200000000001), 'text': ' Yeah, but valid queries for that schema, like can you force to open that query experience even without respect to the schema? Sorry, I kind of.'}, {'timestamp': (735.1200000000001, 739.2800000000001), 'text': " we could just maybe use constraint generation to make sure that at least it opens. It's actually great. We're happy to out even if it doesn't know about the schema you know, but it would be even better to use the constraint generation to with knowledge of the schema. Yeah, definitely."}, {'timestamp': (739.2800000000001, 752.5200000000001), 'text': " I do things like let me show you I think using main I have a drag it's"}, {'timestamp': (752.5200000000001, 756.3600000000001), 'text': " So I first heard Rory is that it doesn't speed like valid. GraphQL are all, but I've never encountered that. Maybe sometimes it's missing like a mutation or query, but the prompts that I'm using are already like covering for that."}, {'timestamp': (756.3600000000001, 762.3600000000001), 'text': ' but in this format, something like that. So for that matter, I use a reggae, a general reggae for refuel queries or mutations. Oh, like an extract part of the result. Yeah, and then I do the validation.'}, {'timestamp': (762.3600000000001, 771.1600000000001), 'text': ' in that front. But yeah, you could like constrain. I think there are some efforts and I dare I read a debate of what was open AI really doing because'}, {'timestamp': (771.1600000000001, 773.6000000000001), 'text': " because you can like feed it back and tell it hey, this is bad, generate again, that's one solution or maybe run it with many temperatures and select the valid ones, that's another solution that I've seen. And finally,"}, {'timestamp': (773.6000000000001, 780.0800000000002), 'text': ' in my ear and then you can like squash the the probability because each like in that layer you will have one one output per tokens so there will be some invalid tokens so you can fill some guy that did that okay.'}, {'timestamp': (780.0800000000002, 789.0800000000002), 'text': " It's trained like only on valid graph QL maybe that would reduce the full capsize. You know what I mean? Yeah, yeah, that's that's kind of the of the of the approach."}, {'timestamp': (789.0800000000002, 794.6000000000001), 'text': " that we are having to select the tool. It's just the training that speeds that. Yeah, they find the model just for the"}, {'timestamp': (794.6000000000001, 802.0800000000002), 'text': " but it has some like small errors. I mean I think I prefer to have a more way more powerful LLM rather than have that from what I've seen it's very easy just to use the"}, {'timestamp': (802.0800000000002, 806.6400000000001), 'text': " the reggae, the function calls are like they kind of intuitive when you read them at a way, right? So yeah, I think like a foundation of model, it's like really helps in that situation. Yeah, I've seen that these models,"}, {'timestamp': (806.6400000000001, 812.0800000000002), 'text': " we can look English to French and it's selected the French model. You see? So there should be, I don't know why I think this is an example in their showcase that they are being honest."}, {'timestamp': (812.0800000000002, 818.0000000000001), 'text': " It just used the model. If you go to the data, they mostly use French, so that's completely biased, it's like overfitted. So I kind of prefer the more general model than trying to, I don't know it's."}, {'timestamp': (818.0000000000001, 825.0000000000001), 'text': " these but for SQL and if you read the paper like very with a lot of attention they didn't gain much by fine tuning. You can find it here. fine tuning maybe."}, {'timestamp': (825.0000000000001, 834.0000000000001), 'text': ' So Google train an SQL model, this palm, they are comparing it with others. So look the figures. The difference between fine-tune and'}, {'timestamp': (834.0000000000001, 839.1200000000001), 'text': ' and a few shots, 60, 7.4, bound with a few shot, so 77.3 to 78.3, 77.3 to 7.3, look.'}, {'timestamp': (839.1200000000001, 846.0800000000002), 'text': " the stuff model to a fine tune model. It was just 1% so I really prefer to have the few shots. That's something that it's also emerging. So once you get the query, you can use also embeddings to get"}, {'timestamp': (846.0800000000002, 848.0800000000002), 'text': " the most of this data is really. It's parts, but I do prefer to use that. And if you see, Langchain, Langchain, let me get Langchain Docs Langchain."}, {'timestamp': (848.0800000000002, 855.0800000000002), 'text': ' Well, they change their docs, but let me show you some examples.'}, {'timestamp': (855.0800000000002, 868.4400000000002), 'text': ' or retreat. Well, they have this concept, you know, that on the basis of the prompt, they fetch the most relevant.'}, {'timestamp': (868.4400000000002, 873.6400000000002), 'text': ' Yeah, by that some point in my make sense to fine tune.'}, {'timestamp': (873.6400000000002, 878.6400000000002), 'text': " I'm looking forward to bringing on my machine and kind of understanding a little bit deeper. And yeah, the exploring more use cases at the sounds really exciting as well."}, {'timestamp': (878.6400000000002, 892.1600000000002), 'text': " So this is basically for CPIT, it works really well because it's small and everything can be"}, {'timestamp': (892.1600000000002, 899.1600000000002), 'text': " Is there a way of this and attempted to deploy this to help with SIP and all is that a feasible thing? I mean yes, because it's not private we can't really do that. Well at this point, just to comply with monadico."}, {'timestamp': (899.1600000000002, 904.2000000000002), 'text': " That's where Sean's ideas about how to like overcome the context when those sides and Local processing would avoid Maybe help if it's if it was female that's it sir. No, it's just using like like a privately host"}, {'timestamp': (904.2000000000002, 919.2000000000002), 'text': ' I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will, I will,'}, {'timestamp': (919.2000000000002, 925.7600000000001), 'text': " information, right? We actually have a pre-processor to resolve what we're seeing on the screen right now. But yeah, there's still some struggles in the local model to get for figure out and get creative with."}, {'timestamp': (925.7600000000001, 931.7600000000001), 'text': ' I do wonder if a summarization focuss could be useful for that token. Contact size prop as well. Like if something I could be applied to this scheme, get it fixed.'}, {'timestamp': (931.7600000000001, 933.7600000000001), 'text': " But I'm going to stop recording. Yeah, I got that. And I appreciate everyone taking time of their day to come here and explore this and kind of help explore how to keep driving."}]} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4bd15883..c983bb1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,12 +41,20 @@ urllib3 yarl==1.9.2 boto3==1.26.151 nltk==3.8.1 -wordcloud -spacy -scattertext -pandas -jupyter -seaborn -matplotlib -termcolor -ffmpeg \ No newline at end of file +wordcloud==1.9.2 +spacy==3.5.4 +scattertext==0.1.19 +pandas==2.0.3 +jupyter==1.0.0 +seaborn==0.12.2 +matplotlib==3.7.2 +matplotlib-inline==0.1.6 +termcolor==2.3.0 +ffmpeg==1.4 +aiortc==1.5.0 +cached_property==1.5.2 +stamina==23.1.0 +httpx==0.24.1 +sortedcontainers==2.4.0 +openai-whisper @ git+https://github.com/openai/whisper.git@248b6cb124225dd263bb9bd32d060b6517e067f8 +https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz diff --git a/server_executor_cleaned.py b/server_executor_cleaned.py new file mode 100644 index 00000000..588b584f --- /dev/null +++ b/server_executor_cleaned.py @@ -0,0 +1,181 @@ +import asyncio +import datetime +import io +import json +import logging +import sys +import uuid +import wave +from concurrent.futures import ThreadPoolExecutor + +import jax.numpy as jnp +from aiohttp import web +from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription +from aiortc.contrib.media import MediaRelay +from av import AudioFifo +from whisper_jax import FlaxWhisperPipline + +from utils import run_in_executor + +logger = logging.getLogger(__name__) + +transcription = "" + +pcs = set() +relay = MediaRelay() +data_channel = None +total_bytes_handled = 0 +pipeline = FlaxWhisperPipline("openai/whisper-tiny", dtype=jnp.float16, batch_size=16) + +CHANNELS = 2 +RATE = 48000 +audio_buffer = AudioFifo() +start_time = datetime.datetime.now() +executor = ThreadPoolExecutor() + + +def channel_log(channel, t, message): + print("channel(%s) %s %s" % (channel.label, t, message)) + + +def channel_send(channel, message): + # channel_log(channel, ">", message) + global start_time + if channel: + channel.send(message) + print( + "Bytes handled :", + total_bytes_handled, + " Time : ", + datetime.datetime.now() - start_time, + ) + + +def get_transcription(frames): + print("Transcribing..") + # samples = np.ndarray( + # np.concatenate([f.to_ndarray() for f in frames], axis=None), + # dtype=np.float32, + # ) + # whisper_result = pipeline( + # { + # "array": samples, + # "sampling_rate": 48000, + # }, + # return_timestamps=True, + # ) + out_file = io.BytesIO() + wf = wave.open(out_file, "wb") + wf.setnchannels(CHANNELS) + wf.setframerate(RATE) + wf.setsampwidth(2) + + for frame in frames: + wf.writeframes(b"".join(frame.to_ndarray())) + wf.close() + global total_bytes_handled + total_bytes_handled += sys.getsizeof(wf) + whisper_result = pipeline(out_file.getvalue(), return_timestamps=True) + with open("test_exec.txt", "a") as f: + f.write(whisper_result["text"]) + whisper_result['start_time'] = [f.time for f in frames] + return whisper_result + + +class AudioStreamTrack(MediaStreamTrack): + """ + An audio stream track. + """ + + kind = "audio" + + def __init__(self, track): + super().__init__() + self.track = track + + async def recv(self): + frame = await self.track.recv() + audio_buffer.write(frame) + if local_frames := audio_buffer.read_many(256 * 960, partial=False): + whisper_result = run_in_executor( + get_transcription, local_frames, executor=executor + ) + whisper_result.add_done_callback( + lambda f: channel_send(data_channel, str(whisper_result.result())) + if (f.result()) + else None + ) + return frame + + +async def offer(request): + params = await request.json() + print("Request received") + offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) + + pc = RTCPeerConnection() + pc_id = "PeerConnection(%s)" % uuid.uuid4() + pcs.add(pc) + + def log_info(msg, *args): + logger.info(pc_id + " " + msg, *args) + + log_info("Created for %s", request.remote) + + @pc.on("datachannel") + def on_datachannel(channel): + global data_channel, start_time + data_channel = channel + channel_log(channel, "-", "created by remote party") + start_time = datetime.datetime.now() + + @channel.on("message") + def on_message(message): + channel_log(channel, "<", message) + + if isinstance(message, str) and message.startswith("ping"): + # reply + channel_send(channel, "pong" + message[4:]) + + @pc.on("connectionstatechange") + async def on_connectionstatechange(): + log_info("Connection state is %s", pc.connectionState) + if pc.connectionState == "failed": + await pc.close() + pcs.discard(pc) + + @pc.on("track") + def on_track(track): + print("Track %s received" % track.kind) + log_info("Track %s received", track.kind) + # Trials to listen to the correct track + pc.addTrack(AudioStreamTrack(relay.subscribe(track))) + # pc.addTrack(AudioStreamTrack(track)) + + # handle offer + await pc.setRemoteDescription(offer) + + # send answer + answer = await pc.createAnswer() + await pc.setLocalDescription(answer) + print("Response sent") + return web.Response( + content_type="application/json", + text=json.dumps( + {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} + ), + ) + + +async def on_shutdown(app): + # close peer connections + coros = [pc.close() for pc in pcs] + await asyncio.gather(*coros) + pcs.clear() + + +if __name__ == "__main__": + app = web.Application() + app.on_shutdown.append(on_shutdown) + app.router.add_post("/offer", offer) + web.run_app(app, access_log=None, host="127.0.0.1", port=1250) diff --git a/server_multithreaded.py b/server_multithreaded.py new file mode 100644 index 00000000..573ace24 --- /dev/null +++ b/server_multithreaded.py @@ -0,0 +1,227 @@ +import asyncio +import datetime +import io +import json +import logging +import os +import sys +import threading +import uuid +import wave +from concurrent.futures import ThreadPoolExecutor +from sortedcontainers import SortedDict +import configparser +import jax.numpy as jnp +from aiohttp import web +from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription +from aiortc.contrib.media import (MediaRelay) +from av import AudioFifo +from whisper_jax import FlaxWhisperPipline + +ROOT = os.path.dirname(__file__) + +config = configparser.ConfigParser() +config.read('config.ini') + +WHISPER_MODEL_SIZE = config['DEFAULT']["WHISPER_MODEL_SIZE"] + +logger = logging.getLogger("pc") +pcs = set() +relay = MediaRelay() +data_channel = None +sorted_message_queue = SortedDict() + +CHANNELS = 2 +RATE = 44100 +CHUNK_SIZE = 256 + +audio_buffer = AudioFifo() +pipeline = FlaxWhisperPipline("openai/whisper-" + WHISPER_MODEL_SIZE, + dtype=jnp.float16, + batch_size=16) + +transcription = "" +start_time = datetime.datetime.now() +total_bytes_handled = 0 + +executor = ThreadPoolExecutor() + +frame_lock = threading.Lock() +file_lock = threading.Lock() + +total_bytes_handled_lock = threading.Lock() + +def channel_log(channel, t, message): + print("channel(%s) %s %s" % (channel.label, t, message)) + +def thread_queue_channel_send(): + print("M-thread created") + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + least_time = sorted_message_queue.keys()[0] + message = sorted_message_queue[least_time] + if message: + del sorted_message_queue[least_time] + data_channel.send(message) + print("M-thread sent message to client") + with total_bytes_handled_lock: + print("Bytes handled :", total_bytes_handled, " Time : ", datetime.datetime.now() - start_time) + except Exception as e: + print("Exception", str(e)) + pass + loop.run_forever() + +# async def channel_send(channel, message): +# channel_log(channel, ">", message) +# if channel and message: +# channel.send(message) + +def get_transcription(local_thread_id): + # Block 1 + print("T-thread -> ", str(local_thread_id) , "created") + global frame_lock + while True: + with frame_lock: + frames = audio_buffer.read_many(CHUNK_SIZE * 960, partial=False) + if not frames: + transcribe = False + else: + transcribe = True + + if transcribe: + try: + print("T-thread ", str(local_thread_id), "is transcribing") + sorted_message_queue[frames[0].time] = None + out_file = io.BytesIO() + wf = wave.open(out_file, "wb") + wf.setnchannels(CHANNELS) + wf.setframerate(RATE) + wf.setsampwidth(2) + + for frame in frames: + wf.writeframes(b''.join(frame.to_ndarray())) + wf.close() + + whisper_result = pipeline(out_file.getvalue()) + + global total_bytes_handled + with total_bytes_handled_lock: + total_bytes_handled += sys.getsizeof(wf) + item = {'text': whisper_result["text"], + 'start_time': str(frames[0].time), + 'time': str(datetime.datetime.now()) + } + sorted_message_queue[frames[0].time] = str(item) + start_messaging_thread() + except Exception as e: + print("Exception -> ", str(e)) + +class AudioStreamTrack(MediaStreamTrack): + """ + A video stream track that transforms frames from an another track. + """ + + kind = "audio" + + def __init__(self, track): + super().__init__() # don't forget this! + self.track = track + + async def recv(self): + # print("Awaiting track in server") + frame = await self.track.recv() + audio_buffer.write(frame) + return frame + + +def start_messaging_thread(): + message_thread = threading.Thread(target=thread_queue_channel_send) + message_thread.start() + # message_thread.join() + +def start_transcription_thread(max_threads): + t_threads = [] + for i in range(max_threads): + t_thread = threading.Thread(target=get_transcription, args=(i,)) + t_threads.append(t_thread) + t_thread.start() + + # for t_thread in t_threads: + # t_thread.join() + +async def offer(request): + params = await request.json() + print("Request received") + offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) + + pc = RTCPeerConnection() + pc_id = "PeerConnection(%s)" % uuid.uuid4() + pcs.add(pc) + + def log_info(msg, *args): + logger.info(pc_id + " " + msg, *args) + + log_info("Created for %s", request.remote) + + @pc.on("datachannel") + def on_datachannel(channel): + global data_channel, start_time + data_channel = channel + channel_log(channel, "-", "created by remote party") + start_time = datetime.datetime.now() + + @channel.on("message") + def on_message(message): + channel_log(channel, "<", message) + if isinstance(message, str) and message.startswith("ping"): + # reply + channel.send("pong" + message[4:]) + + @pc.on("connectionstatechange") + async def on_connectionstatechange(): + log_info("Connection state is %s", pc.connectionState) + if pc.connectionState == "failed": + await pc.close() + pcs.discard(pc) + + @pc.on("track") + def on_track(track): + print("Track %s received", track.kind) + log_info("Track %s received", track.kind) + # Trials to listen to the correct track + pc.addTrack(AudioStreamTrack(relay.subscribe(track))) + # pc.addTrack(AudioStreamTrack(track)) + + # handle offer + await pc.setRemoteDescription(offer) + + # send answer + answer = await pc.createAnswer() + await pc.setLocalDescription(answer) + print("Response sent") + return web.Response( + content_type="application/json", + text=json.dumps( + {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} + ), + ) + + +async def on_shutdown(app): + # close peer connections + coros = [pc.close() for pc in pcs] + await asyncio.gather(*coros) + pcs.clear() + + +if __name__ == "__main__": + app = web.Application() + app.on_shutdown.append(on_shutdown) + start_transcription_thread(6) + app.router.add_post("/offer", offer) + web.run_app( + app, access_log=None, host="127.0.0.1", port=1250 + ) + + diff --git a/streamclient.py b/streamclient.py new file mode 100644 index 00000000..f6f12e8c --- /dev/null +++ b/streamclient.py @@ -0,0 +1,168 @@ +import asyncio +import logging +import time +import uuid +import threading +import configparser +import httpx +import pyaudio +import requests +import ast +import stamina +from aiortc import (RTCPeerConnection, RTCSessionDescription) +from aiortc.contrib.media import (MediaPlayer, MediaRelay) + +logger = logging.getLogger("pc") +file_lock = threading.Lock() + +config = configparser.ConfigParser() +config.read('config.ini') + +class StreamClient: + def __init__( + self, + signaling, + url="http://127.0.0.1:1250", + play_from=None, + ping_pong=False, + audio_stream=None + ): + self.signaling = signaling + self.server_url = url + self.play_from = play_from + self.ping_pong = ping_pong + self.paudio = pyaudio.PyAudio() + + self.pc = RTCPeerConnection() + + self.loop = asyncio.get_event_loop() + # self.loop = asyncio.new_event_loop() + self.relay = None + self.pcs = set() + self.time_start = None + self.queue = asyncio.Queue() + self.player = MediaPlayer(':' + str(config['DEFAULT']["AV_FOUNDATION_DEVICE_ID"]), + format='avfoundation', options={'channels': '2'}) + + def stop(self): + self.loop.run_until_complete(self.signaling.close()) + self.loop.run_until_complete(self.pc.close()) + # self.loop.close() + print("ended") + + def create_local_tracks(self, play_from): + if play_from: + player = MediaPlayer(play_from) + return player.audio, player.video + else: + if self.relay is None: + self.relay = MediaRelay() + print("Created local track from microphone stream") + return self.relay.subscribe(self.player.audio), None + + def channel_log(self, channel, t, message): + print("channel(%s) %s %s" % (channel.label, t, message)) + + def channel_send(self, channel, message): + # self.channel_log(channel, ">", message) + channel.send(message) + + def current_stamp(self): + + if self.time_start is None: + self.time_start = time.time() + return 0 + else: + return int((time.time() - self.time_start) * 1000000) + + async def run_offer(self, pc, signaling): + # microphone + audio, video = self.create_local_tracks(self.play_from) + pc_id = "PeerConnection(%s)" % uuid.uuid4() + self.pcs.add(pc) + + def log_info(msg, *args): + logger.info(pc_id + " " + msg, *args) + + @pc.on("connectionstatechange") + async def on_connectionstatechange(): + print("Connection state is %s" % pc.connectionState) + if pc.connectionState == "failed": + await pc.close() + self.pcs.discard(pc) + + @pc.on("track") + def on_track(track): + print("Sending %s" % track.kind) + # Trials + self.pc.addTrack(track) + # self.pc.addTrack(self.microphone) + + @track.on("ended") + async def on_ended(): + log_info("Track %s ended", track.kind) + + self.pc.addTrack(audio) + + # DataChannel + channel = pc.createDataChannel("data-channel") + self.channel_log(channel, "-", "created by local party") + + + async def send_pings(): + while True: + self.channel_send(channel, "ping %d" % self.current_stamp()) + await asyncio.sleep(1) + + @channel.on("open") + def on_open(): + if self.ping_pong: + asyncio.ensure_future(send_pings()) + + @channel.on("message") + def on_message(message): + self.queue.put_nowait(message) + if self.ping_pong: + self.channel_log(channel, "<", message) + + if isinstance(message, str) and message.startswith("pong"): + elapsed_ms = (self.current_stamp() - int(message[5:])) / 1000 + print(" RTT %.2f ms" % elapsed_ms) + + await pc.setLocalDescription(await pc.createOffer()) + + sdp = { + "sdp": pc.localDescription.sdp, + "type": pc.localDescription.type + } + + @stamina.retry(on=httpx.HTTPError, attempts=5) + def connect_to_server(): + response = requests.post(self.server_url, json=sdp, timeout=10) + response.raise_for_status() + return response + + params = connect_to_server().json() + answer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) + await pc.setRemoteDescription(answer) + + self.reader = self.worker(f"worker", self.queue) + + def get_reader(self): + return self.reader + + async def worker(self, name, queue): + while True: + msg = await self.queue.get() + msg = ast.literal_eval(msg) + with file_lock: + with open("test_sm_6.txt", "a") as f: + f.write(msg["text"]) + yield msg["text"] + self.queue.task_done() + + async def start(self): + print("Starting stream client") + coro = self.run_offer(self.pc, self.signaling) + task = asyncio.create_task(coro) + await task diff --git a/transcript_timestamps(2).txt b/transcript_timestamps(2).txt deleted file mode 100644 index 98b876ec..00000000 --- a/transcript_timestamps(2).txt +++ /dev/null @@ -1 +0,0 @@ -{'text': " So everything's going well so far. All of that. Okay, fine. I'm going to join us. This is where you keep this low key. Very casual. Absolutely. All right. Well, thank you all for being here. We're very honored that you came to join us for this conversation. We are going to spend a little bit of time. I know it was written up, right? But thinking about how you actually grow and retain your employees, we know that there's a war on talent, especially in the cyberspace right now, right? Everybody's trying to get everybody in the door. Type lines are a little bit dry, a little bit hard to find. It's a tricky scenario. What we want to do if you allow us for this session is to almost park that call to the site. We're just going to spend just leaving for the moment. We're just going to put that aside. And instead, what we're going to focus on, or the employees that you actually have on board, already. We know that the hiring piece is complex. It requires dollars and HR and a whole bunch of stuff. That's that. We're going to focus on the rest, which is your workforce today. How do you grow them? How do you retain them? And in doing so, you actually find the bill become more effective and efficient and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go the bell become more effective and efficient and committed to your organization. So then they'll go ahead and become random ambassadors for youis. I am a senior manager in the WIC capital practice. I'm focus on organizational transformation. What does that mean? That's where we quite all of our organizational design work. So how offices are aligned on the work chart and how information and decisions flow. In addition, it's where we hold our culture and our communications work. I've spent my career at Lloyd and beyond previously focused on culture and communications and employee and workforce programs and how you encourage the engagement and employee experience is the best thing to do. I've done that in a couple of clients' faces including the Federal Space, State Space, Global Space, including with some cyber agencies as well. But that'll answer my question. Thanks, Seize. You're going to have to be in the everyone. I'm part of the task. And I'm also a senior manager, absolutely. And I lead work for six periods for the government and public services practice at our firm. I want to forefront of workforce experience issues. For commercial cities, state, and federal clients focusing on training, attracting talent, recruiting, retention, and really today what we want to do is share some best practices, tools, what we've seen from other clients and help you. you some ideas about how to become an employer and choice, and how to retain the attract, high-performing cyber-talented. So, a little divergent? Before we do, I just want to give one caveat. Because it's day two of the conference now that the feedback cyber-fert, a while. I would say Mar and I both want to copy up the neater of us, or going to be cyber experts, especially in the room of this caliber. Yeah, right? We do this with cyraclients, but we also use with other clients. And so our invitation to you is to take what we're going to share here and think about how that applies to your day-to-day organization. So we kind of did this a little bit to stretch that thinking outside of just this formal cyber realm and bring you some ideas of what we're seeing from outside. That works. Okay. Great. So, actually, we define work for six periods, holistically, as there's some of our workers lived experience at work and how they feel about their organization. And so, really, this is shaped by eight key dimensions that impact worker experience overall. So we'll start by walking through each of these. So really this framework is backed by research as well as testing solutions with both our commercial as well as our government clients and we start with the people I work with. And this is really support and recognition for managers and you know who you interact with day to day. That's all in the side. It has one of the top five factors or talent retention. for the technology that I use. So that's very much about frustration, free technology. So are you having to do work around? So you're having to call IT or spend a lot of your day trying to figure out how to actually use this technology to do your job. And are there ways to use technology to improve collaboration, coordination and communication in a way that you're able to do your work more efficiently. The next little focus on is the places I do work, which is really focusing on the flexibility of the physical workspace and how that improves employee productivity, increases job satisfaction, and lowers overall workplace stress. Another one that we focus on is a sensible longing and or theness. That an organization creates. So that's very much about increasing your job performance. I think we've cited that it can lead to a 56% increase in job performance and a 50% reduction in turnover risk. And the work I do. I think this is particularly important to cyber employees where a lot of government employees are driven by the mission of the work that they're doing. And so connecting work to meaningful experiences and reiterating that's fulfilling and contributes to something bigger themselves is a great way to motivate and keep employees, that increases workers satisfaction. And then another one is really focusing on the mission. So identifying, you know, the organizations purpose and connecting it to their own personal values, as well as the mission of the organization. And finally, education. So this is really about higher form of organizations. And the fact that we have a lot of opportunities, we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot the mission of the organization. And finally, education. So this is really about high-performing organizations and the fact that 30 times, 37 times more likely, workers are going to be able to achieve their local roles and stay within organization if they have the proper training and support and experiences to help them get there. So really when thinking about workforce experience, organizations, you know, often are sure where to start or what elements impact their employees. So this framework is really what we use as a starting point to help orient our clients and also help them prioritize certain employee activities and relationships. So these are the key contributors that we've seen in our experience to employees really engage and motivated, which has directly contributed to retaining a attracting talent. So with that we are going to move on to the next slide. So just our off cybersecurity employees we know are experiencing disruption unlike ever before. And so just to a great day's discussion, we first went to start by walking through how one agency has approached the issue and used work force experience to address it. So a large federal agency of the Agile Cyber Security Priorities had long-suning workforce issues, you know, lots of attrition, tough time attracting the right talent for the roles that they needed. And so, they really double-attacking and focusing on career growth and development, culture, recognition, route, and investing in trust in leadership. So as a result of reimagining the workforce experience, they were able to retain critical employees for a major transition. And a few things that they saw as a result of investment in workforce experience programs include the increase of frequency of career focus conversations, increased in informal recognition across the entire organization, and the employees really felt that they had the tools to help them to do their job center, and also to prevent burnout amongst their teams, which was an issue at this organization that is facing. So next, I'll pass it over to Susie and to speak about the cyberwork for today. Probably a little bit about what you all are seeing. Thanks, Mara. So there are a couple of stats up on this slide. I don't think they're going to surprise anybody, but I just want to call them out, because I think they're an important framing for what we are seeing in this library industry, of course, which you're all experts in. So in 2022 last year at the Boyd International Association of State CIOs, to Disciber or security study is anyone familiar with the study? Can you see it? No, I don't know. Okay, I'll go through a learn after reading for you. Yeah. And so it was a survey of all 50 state C-Sos and three territories C-Sos. So very comprehensive across the United States. And they were asking questions about the general cybersecurity environment, but in addition, some things came out around the workforce. So when those CISO's were asked about how they can support emerging threats and what their biggest challenges, number one, they said legacy infrastructure and solutions, number two, they said workforce. The number two thing, holding these state CISO's back from responding to the threats that are coming is the workforce. That's a pretty big issue. And the number two thing, the number two thing, the number two thing, holding these state CISO's back from responding to the threats that are coming is the workforce. That's a pretty big issue. And the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, holding these state-sea-sodes back from responding to the threats that are coming, is the workforce. That's a pretty big issue. In addition, 62% out-to-thirds reported that their staff have a gap in competencies. Don't have the skills to do the job. Another pretty big issue when you step back and look at it. When we talk about the belonging piece that Mara just talked about and shared, 23% with the respondents didn't know if their organization has established DEI leadership positions. So is there someone in the organization really focused on that inclusion piece and bringing all of your workforce along? And only 25% reported offering remote work options, which I know is tricky in the cyberspace. When you think about the sock and everything else, but it is there are ways to do it with at least parts of the workforce. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And I know is tricky in the cyberspace. When you think about the sock and everything else, but it is there are ways to do it with at least parts of the workforce. And we know that that's super important to the millennials and slunder-reppers any groups. Remote work has certainly been a topic since the pandemic, right? I see people nodding. It's just on everybody's tongue, it's front of mind. So how do you make that work? You know, addition to all of this, I think what's really important to know, right? Is that cyber threats have not produced in any way? So there's one stat that says that as a result of COVID-19, it's been linked to a 238% increase in cyber crime. So the numbers are going up the need continues and the resourcing and the workforce, maybe as a little bit lagging behind. In addition to this in a cyberspace, we're also seeing some pretty big macro trends in workforce space in general that are contributing to some of the challenges going on today. And these also probably won't surprise you. Number one, exhaustion. So this is where that burnout topic that some of you people are talking about is showing up. And in cyber agencies, we have frontline workers, right? There's a 24-7 operation. And so you're going to have this burnout challenge unfortunately. This is also contributed by working from home in some ways and that blurring of that work life home life when it ended the day happens if it happens and whose days on their laptop all night working. In addition, between 2021 and 2022, the number of meetings we all experience increased 280 percent. So you're getting your Zoom fatigue, your Teams fatigue, right? Does that meetings fatigue along the way? Resignation? Who's heard of the great resignation coming out of the pandemic? Yeah. So that's where the topic says it. We know that a lot of people left the workforce voluntarily and in fact in 2021 more people than ever before left the workforce according to the Bureau of Labor Statistics. Some people have come back and some people have it. So now our options for who's out there working and interested in being in the workforce and talent diversity in that they bring to your organizations, give you a little more limited. And then reshuffling. 34% of US workers shifted their cities since the start of COVID. 34% of the population is in the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of US workers shifted their cities since the start of COVID. 34, a third of US workers have moved cities. And so this is where that remote work conversation becomes so important about can you get the right people. And so the people again that you have on board, how are you engaging them, keeping them, working in their most effective to support themselves in your organization? Why does all this matter? Because we know that losing employees is costly. Giving them a positive workforce experience means that they stay with your organization, they are committed, they are loyal, and they continue to grow and build within your organization. One study shows that losing a employee and having to recruit a new one, cost three to four times their salary. Yeah, opportunity costs the knowledge drain, the process of just onboarding the job up on the boards, bringing them on three to four times their salary. So if you can get it right on the front end and really have that workforce experience be positive and keep your people it's going to pay off dividends in the actual. I'm going to pass it back to Mara who's going to tie this back to that model for us. I mean, building on what Susie said workforce experience has never been more important given this crisis and the urgency around keeping talented and finding the right talent. Since the workforce is an organization's host valuable asset, investing in workforce experience activities, we found because lead to more productive work, more efficient work, more innovative approaches to the work, and more engaged teams which ultimately results in better mission outcomes for your organization. And we found a direct correlation with the ability to retain employees, improve efficiency overall in your organization and the organization changes and transitions more effectively. From our work with other agencies as well as commercial organizations, we found that investing in work more experienced is the most sustainable way to create a competitive advantage that improves the business and talent outcomes. And so, it's investing in work more experience is a single best way that organizations can minimize workforce disruption in the short term. I'm going to get ahead of future work. How do we do that? And so in this next section we're going to focus on how cyber employers can win at workforce experience. So based on a 2022 state CIO, mass CO study that included 51 participants, it was identified that the following areas were some of the top priorities around how to improve workforce experience. work options as particularly important 8% focus on flexible work schedules and how work affects their life 35% focus on how I grow as a human so focusing on reskilling opportunities for rotation programs and through the right training to make sure that the workforce is able to meet the needs of the modern IT to man's. And so with this, Suzy is going to walk us through a deeper dive of the different options on how to address these results with this actually might look like in practice. Great. Thank you, Mara. So again, just to take it from the top, right? You know that the talent pipeline is tough. We know that we're recruiting processes challenging. I think a little deeper into some of these things. We're going to walk through a couple of activities, programs, initiatives, ideas, just to hopefully give you something to walk away with today that you can take back to your organization and implement. And the idea behind these is you can implement them tomorrow. You don't need to get into an IT backlog of some system build. You don't need to go to HR and say, I need, you know, millions of dollars to hire people and fill the gaps. These are things that you can do with your current workforce in place today to improve that workforce experience. So, starting with how I grow with a human and again 35% of the CIOs said that this was one of the most important things. A couple of ideas here for things that can be done and things that we've seen these accessible elsewhere. Number one, integrating training into the day-to-day. This is not about sending it employee off to go take a training, get a certification, step out of the workplace, and come back. We know that most adult learners prefer experiential learning. And so the way to do that is a apprenticeship, mentorship, shadowing, rotational programs, lunch and learns, informal training opportunities, action, after action reports where you can also around and talk about what did we do well, what didn't we do, what are we going to do differently. And some of this also is just documenting what it is you do. So setting up some of those SOPs so that you can pass those over to somebody and say, well this is the best way to do it. Here's a lesson's learned. Identifying opportunities for employees to use their strengths daily. When we talk about strengths, we don't just mean what am I good at. What we mean by strengths is what gives me individually energy. What energizes me? What excites me? And so I may be very good at data analytics, but that doesn't mean that that's my passion. I may actually prefer competing teams or communications or something like that. And so checking in with your teams about what is it that they each person enjoys most and trying to align them to that work. Studies have shown that if you align people to their strengths, again, the things that energize them, they will actually outperform, and do way better than if you give them a challenge task or activity that then they have to sort of overcome and be able to accomplish. We are good when we're playing to our strengths. So it doesn't mean that in a play you can only do their strength all day and be happy, but try to find ways for them to incorporate their strengths into the day to day as much as possible. And then another thing that tends to be useful is stepping back and building looking at career pathways. So how are you going to grow people from level to level, from ability to ability? Is anybody here familiar with the nice framework? Come on, Nick. Yeah, a couple of hands. So this is a great resource if you haven't seen an highly recommend going onto the website. They have work titles, they have job descriptions, they have competencies, they have skills, knowledge, experiences, and you can stack roles so you can say, okay, if I'm an IT leadership management position today, and I want to become a data analyst, and they will help you kind of, they map that path a little bit for you with the overlap between those two roles for example and then some of the case this knowledge skills and abilities to go get from one to the other. So a very very useful tool used by a lot of cyber agencies and organizations highly highly highly highly recommend using it or at least checking an out and seeing if there's anything useful for you there. But really looking at where are my people going and how to like grow them because we know also that one of the most important things that we have to do is to make sure that we have a lot of information about the data that we have to do with the data and that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it environment. I know the sock is what it is. We know that piece. But there are conversations still to be having around work location. What we have found is most successful is when organizations really focus in on work location. In alignment to your desires, your goals, your strategy. There are some things that working remote are great for. Heds downtime, idea generation, focus, and productivity. There are some things that working in person are great for. Teamwork, apprenticeship, and some of those sort of incidental connection points, running into somebody, you know, by the water cooler and finding out that we're doing similar things, and we can train notess actually Mara and I just did that and I said oh send me that if you could because I want to see that right? So designing intentionally for what work can be done in person, what work can be done remotely, maybe it's part and part. Maybe there's a single initiative or project where you can send people to go home and work remotely and then come back and they're still feeling like they're getting a taste of that opportunity as they're normally meeting to be on site. So really being smart and intentional and purposeful about how you design that. Looking at the digital communications and information sharing. So a lot of us have moved on to teams and Slack and Zoom and these digital tools. Are you really using them for their maximum capability to support your organization? Is there somebody in your organization who really is tasked with understanding all of the features that you have license to and bringing that goodness to your organization to help your people work even better? And then the third one there is forced to connectivity and networking especially in a hybrid or remote environment. We know that happiness at work and elsewhere is tied to social connection. So how are you forcing some of that social connection? Maybe it is in those brown bag trainings or after action reports. Maybe it is taking a moment before diving into a meeting and saying, hey, how is everybody's weekend? Or what are you doing this weekend? Or tell me something funny? Or if you were going to be a office supply, what would you be? Whatever it may be, right? But finding a way to sort of force a little door opening and having people build that connection. And then the well being piece. And so I shared with you about the exhaustion, you know, and the burnout rates and all that earlier. A lot of organizations, especially since during and since the pandemic, have stood up well being activities. They're not necessarily aligned to a situation where they're not going to be able to do that. And so I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to do that. And I'm going to be able to do that. And I'm going to do that. And I'm going to do that. And I'm going to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that. So it's not just sort of a scatter plot of activities, but there's sort of intentional and organized aligning back to those goals that you have. And then, remember that impact of that and adjust as needed. And if you're engaging your workforce correctly, then you'll have lots of data points about what's working and what's not in order to make those adjustments. make those adjustments. Started on this a little bit, but again, thinking creatively, not just about remote work, but how shift work is scheduled. So a lot of the conversation around workforce and economic development is around people who may be to be taking their child care duties at home while still trying to get into the workforce. What is it that you can do to maybe look at four hour shifts or six hour shifts different than your typical shifts to bring in more people in diversity, and also alleviate some of the burnout and pressure that's happening for the workforce in place today. There are a bunch of apps that focus on this. A lot of them are focusing in the health care space of the transportation frontline workers space, but they could easily be leveraged over into the cyberspace as well. And then the third one here conducting a culture assessment. A lot of organizations assume that culture sort of just happens and it doesn't. It's important to be intentional about culture and what do we mean when we talk about culture, right? Because we all sort of live in it all the time, but what is that? When we talk about culture, what we talk about are the intangibles, your beliefs and your values as an organization. Are we more individually individualistic, the ability to talk about culture, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about culture, what we talk about are the intangibles, your beliefs and your values as an organization. Are we more individually individualistic, focused or are we more collaborative? Are we more risk of first? Or are we more innovative and risk takers? Are we more internally focused or do we have external customers that we've really organized around? And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be. And what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be. And what are the steps to get from a to the other. And so when you do a, what are the steps to get from a to the other. And so when you do a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, where you want your organization to be, and what are the steps to get from A to Z? And how do you do that in a way that then filters and flows through your entire organization? So that by the time you get to performance reviews or your next strategic planning cycle, we're standing up a new program or redesigning your organization, there is an friction with what it is you really want from your employees and what you're trying to achieve as an organization and how you're getting there. So cultural assessment, well, it seems like a sort of unnecessary. It's actually very effective and super supportive of your articles. Okay, I've talked a lot. Lots of ideas. Hopefully, good menu for you to pick at least something from a fithesical. I'm going to pass it back to Mara to kind of bring it all together for us. Thank you, Sissy. So, how can employers win at work for six variants? So, practically speaking, they keep it implementing a successful work for six variants. Really starts with understanding what your workforce wants. That can be through a survey or focus groups doing doing user research, but really putting their voice at the center of these initiatives. And then working to prioritize which ones within your organization and help you achieve your overall business objectives. So we're going to walk through our perspective on what it means to really bring work for six periods to an organization. So starting with employers need to think about beyond employment engagement. And so what we mean by this is it's not just one data point, our employees engaged, you know, how I retained my staffer did I recruit an attractive, you know, this number of individuals? It's beyond that and a combination of those aid characteristics that really helps to retain employees and attract the right talent and enhance the overall workforce experience. So an example of that is, you know, looking at physical workspace. So, you know, do they are they able to do their work remotely? In some cases, as Susie mentioned, they do they have everything they need in the office in order to be productive, to get things done in terms of head-stallam work versus hue collaboration, culture, do they, are they recognized for their work? It was great as a hundred dollar gift card is for a great job and something. It in a lot of cases what we found is it's equally just as advantageous to how we manage our reach out and saying thank you so much. You did a great job and acknowledge that. And that's why we know what can motivate and keep employees engaged. And another area is technology. I think we mentioned you know it's a way to help collaborate, work remotely, but it can also be an impediment to when technology isn't easily accessible within organization. There are challenges, IT is kind of available to help with certain things, but how would you approach across those eight dimensions multiple areas of prioritizing different initiatives? The second one that we really focus on and this goes back to its Susie set of Fiesta's and also what I mentioned around the voice of the workforce. And so, incorporating that throughout the design is incredibly important. One of the first things that we often do is pull survey of a particular group to understand what they really hear about. What are their preferences? If they had, it's not just, I want more money, but but rather if you had a choice between a little bit more flexibility in your work schedule versus you know increased recognition, you know, where do they release stand in terms of what should actually get prioritized? And when you drill down into the art of the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work even, you know, communicating that you're doing this and engaging the workforce early on and incorporating them into this because this is important to the organization and to you as leaders is an important way. I think also to retain talent. The third one is a continuous listing approach and this one really focuses on not just pulsing a workforce once a year through an annual HR survey of how do you really feel like you know what leadership considerations should we implement or you know how can we enhance the performance management process? It's really on an ongoing basis and even in in an informal way, you as leaders and managers of teams asking your teams, you know, what they care about, what their frustrated about, what their preferences are, and taking small steps where you can to try to incorporate that into how they work. And you know, sometimes it's challenging to do organization-wide changes that take a lot of funding and investment and capabilities. So often, you know, you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, sometimes it's challenging to do organization why changes that take a lot of funding, investment, and capabilities. So oftentimes, a way to do that is through, you know, more informal touch points on individual teams that you're leading. We found that that's a concept that's a quick way to be able to get a lot of employee engagement and routine and keep your talent happy. The final one is workplace technology and we've talked about this a little bit and I think really with this it's how do you boost productivity? Workers want to do a great job. They're they're at the perform well. They, you know, are often very mission-driven. They want to grow their careers and they want to try to be as efficient as they can in their job. But oftentimes, the organization decides in terms of technology versus what the worker actually needs. There's a bit of a disconnect. And so really looking at how this can give you a major accelerator based on how you work. Do you want an hands collaboration? Do you want to try to create more opportunities for a hybrid or remote work? And, you know, through technology, that can really enhance and accelerate a lot of those activities. And so with all of these four areas combined, this is what we've seen across, you know, four-Jet 500 companies, federal agencies, city, city, agencies as well, as kind of the key characteristics and commonalities amongst the most effective and fully engagement and workforce experience programs. And just to kind of round things out with this globe is, you know, if workforce experience, I recognize, you know, can seem kind of fluffy and like, oh, that's a nice to have. But our premise is that your workforce is the most important asset in your organization. And if they are in the most important asset, you really want to invest in them. To, you know, day to day, be the driver of change in, you know, be the lower productivity. We've just found that, you know, by investing in this, you know, day to day be the driver of change in, you know, be the alert productivity. We've just found that, you know, by investing in this and putting the workforce as, you know, the center part of what you invest in as an organization in leaders. It's not only about retention talent, you know, the cyber workforce crisis, but people want to do work well and they're able to get more done and achieve more without you, you know, directly supervising and micro managing or looking at everything because they just genuinely want to work for you and are able to be more productive overall in that way. So with that, that is our presentation for today. I hope there's a little bit of, you know, the landscape of the cyber workforce with some practical tips that you can take away for how to just think about, you know, improving the overall workforce experience and investing in your employees. So with this, you know, we know that all of you are in the trenches every day. You're facing this, you're living this. And we are just interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going get someone. Sorry. Don't be shy. So yeah, I'm so worried. How is something or what hasn't worked well, you know, that's something that's easier to start with too. But we'd love to just hear from all of you because I think, you know, how we've aggregated a lot of these best practices and what we've come with, it is by, you know, hearing other experiences from other organizations. And so, one of the best and most effective things, I feel like I take away from conferences often is hearing from many peers and what they're facing and, you know, let, you know, similarly, I could bring back to what I'm leading. What about you? Oh, I see him back there. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I Thank you.", 'chunks': [{'timestamp': (0.0, 2.0), 'text': " So everything's going well so far."}, {'timestamp': (2.0, 4.0), 'text': ' All of that. Okay, fine.'}, {'timestamp': (4.0, 8.0), 'text': " I'm going to join us. This is where you keep this low key. Very casual."}, {'timestamp': (8.0, 15.0), 'text': " Absolutely. All right. Well, thank you all for being here. We're very honored that you came to join us for this conversation."}, {'timestamp': (15.0, 18.0), 'text': ' We are going to spend a little bit of time. I know it was written up, right?'}, {'timestamp': (18.0, 22.0), 'text': ' But thinking about how you actually grow and retain your employees,'}, {'timestamp': (22.0, 28.88), 'text': " we know that there's a war on talent, especially in the cyberspace right now, right? Everybody's trying to get everybody in the door. Type lines are a"}, {'timestamp': (28.88, 34.08), 'text': " little bit dry, a little bit hard to find. It's a tricky scenario. What we want to do if you allow"}, {'timestamp': (34.08, 40.56), 'text': " us for this session is to almost park that call to the site. We're just going to spend just"}, {'timestamp': (40.56, 115.0), 'text': " leaving for the moment. We're just going to put that aside. And instead, what we're going to focus on, or the employees that you actually have on board, already. We know that the hiring piece is complex. It requires dollars and HR and a whole bunch of stuff. That's that. We're going to focus on the rest, which is your workforce today. How do you grow them? How do you retain them? And in doing so, you actually find the bill become more effective and efficient and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go ahead and become more effective and committed to your organization. So that then they'll go the bell become more effective and efficient and committed to your organization. So then they'll go ahead and become random ambassadors for youis. I am a senior manager in the WIC capital practice. I'm focus on organizational transformation. What does that mean? That's where we quite all of our organizational design work. So how offices are aligned on the work chart and how information and decisions flow. In addition, it's where we hold our culture and our communications work. I've spent my career at Lloyd and beyond previously focused on culture and communications and employee and workforce programs and how you encourage the engagement and employee experience is the best thing to do."}, {'timestamp': (115.0, 148.08), 'text': " I've done that in a couple of clients' faces including the Federal Space, State Space, Global Space, including with some cyber agencies as well. But that'll answer my question. Thanks, Seize. You're going to have to be in the everyone. I'm part of the task. And I'm also a senior manager, absolutely. And I lead work for six periods for the government and public services practice at our firm. I want to forefront of workforce experience issues. For commercial cities, state, and federal"}, {'timestamp': (148.08, 156.56), 'text': ' clients focusing on training, attracting talent, recruiting, retention, and really today what we want to do is'}, {'timestamp': (156.56, 168.0), 'text': " share some best practices, tools, what we've seen from other clients and help you. you some ideas about how to become an employer and choice,"}, {'timestamp': (168.0, 176.0), 'text': ' and how to retain the attract, high-performing cyber-talented. So, a little divergent? Before we do, I just want to give one caveat.'}, {'timestamp': (176.0, 182.0), 'text': " Because it's day two of the conference now that the feedback cyber-fert, a while. I would say Mar and I both want to"}, {'timestamp': (182.0, 260.0), 'text': " copy up the neater of us, or going to be cyber experts, especially in the room of this caliber. Yeah, right? We do this with cyraclients, but we also use with other clients. And so our invitation to you is to take what we're going to share here and think about how that applies to your day-to-day organization. So we kind of did this a little bit to stretch that thinking outside of just this formal cyber realm and bring you some ideas of what we're seeing from outside. That works. Okay. Great. So, actually, we define work for six periods, holistically, as there's some of our workers lived experience at work and how they feel about their organization. And so, really, this is shaped by eight key dimensions that impact worker experience overall. So we'll start by walking through each of these. So really this framework is backed by research as well as testing solutions with both our commercial as well as our government clients and we start with the people I work with. And this is really support and recognition for managers and you know who you interact with day to day. That's all in the side. It has one of the top five factors or talent retention."}, {'timestamp': (288.0, 311.0), 'text': " for the technology that I use. So that's very much about frustration, free technology. So are you having to do work around? So you're having to call IT or spend a lot of your day trying to figure out how to actually use this technology to do your job. And are there ways to use technology to improve collaboration, coordination and communication in a way that you're able to do your work more efficiently. The next little focus on is the places I do work, which is really focusing on the flexibility of the physical workspace and how that improves employee productivity, increases job satisfaction, and lowers overall workplace stress. Another one that we focus on is a sensible longing and or theness."}, {'timestamp': (311.0, 406.0), 'text': " That an organization creates. So that's very much about increasing your job performance. I think we've cited that it can lead to a 56% increase in job performance and a 50% reduction in turnover risk. And the work I do. I think this is particularly important to cyber employees where a lot of government employees are driven by the mission of the work that they're doing. And so connecting work to meaningful experiences and reiterating that's fulfilling and contributes to something bigger themselves is a great way to motivate and keep employees, that increases workers satisfaction. And then another one is really focusing on the mission. So identifying, you know, the organizations purpose and connecting it to their own personal values, as well as the mission of the organization. And finally, education. So this is really about higher form of organizations. And the fact that we have a lot of opportunities, we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot of opportunities to do that. And we have a lot the mission of the organization. And finally, education. So this is really about high-performing organizations and the fact that 30 times, 37 times more likely, workers are going to be able to achieve their local roles and stay within organization if they have the proper training and support and experiences to help them get there. So really when thinking"}, {'timestamp': (406.0, 412.56), 'text': ' about workforce experience, organizations, you know, often are sure where to start or what elements'}, {'timestamp': (412.56, 418.08), 'text': ' impact their employees. So this framework is really what we use as a starting point to help'}, {'timestamp': (418.08, 426.12), 'text': ' orient our clients and also help them prioritize certain employee activities and relationships. So these are the key'}, {'timestamp': (426.12, 432.56), 'text': " contributors that we've seen in our experience to employees really engage and motivated, which has"}, {'timestamp': (432.56, 476.0), 'text': " directly contributed to retaining a attracting talent. So with that we are going to move on to the next slide. So just our off cybersecurity employees we know are experiencing disruption unlike ever before. And so just to a great day's discussion, we first went to start by walking through how one agency has approached the issue and used work force experience to address it. So a large federal agency of the Agile Cyber Security Priorities had long-suning workforce issues, you know, lots of attrition, tough time attracting the right talent for the roles that they needed."}, {'timestamp': (476.0, 526.16), 'text': " And so, they really double-attacking and focusing on career growth and development, culture, recognition, route, and investing in trust in leadership. So as a result of reimagining the workforce experience, they were able to retain critical employees for a major transition. And a few things that they saw as a result of investment in workforce experience programs include the increase of frequency of career focus conversations, increased in informal recognition across the entire organization, and the employees really felt that they had the tools to help them to do their job center, and also to prevent burnout amongst their teams, which was an issue at this organization that is facing. So next, I'll pass it over to Susie and to speak about the cyberwork"}, {'timestamp': (526.16, 526.8), 'text': ' for today.'}, {'timestamp': (526.8, 529.6), 'text': ' Probably a little bit about what you all are seeing.'}, {'timestamp': (529.6, 530.52), 'text': ' Thanks, Mara.'}, {'timestamp': (530.52, 532.0), 'text': ' So there are a couple of stats up on this slide.'}, {'timestamp': (532.0, 533.72), 'text': " I don't think they're going to surprise anybody,"}, {'timestamp': (533.72, 534.76), 'text': ' but I just want to call them out,'}, {'timestamp': (534.76, 536.56), 'text': " because I think they're an important framing"}, {'timestamp': (536.56, 538.8), 'text': ' for what we are seeing in this library industry,'}, {'timestamp': (538.8, 540.96), 'text': " of course, which you're all experts in."}, {'timestamp': (540.96, 544.52), 'text': ' So in 2022 last year at the Boyd International Association'}, {'timestamp': (544.52, 548.0), 'text': ' of State CIOs, to Disciber or security study is anyone familiar with the study?'}, {'timestamp': (548.0, 550.0), 'text': ' Can you see it?'}, {'timestamp': (550.0, 551.0), 'text': " No, I don't know."}, {'timestamp': (551.0, 554.0), 'text': " Okay, I'll go through a learn after reading for you."}, {'timestamp': (554.0, 555.0), 'text': ' Yeah.'}, {'timestamp': (555.0, 561.0), 'text': ' And so it was a survey of all 50 state C-Sos and three territories C-Sos.'}, {'timestamp': (561.0, 563.0), 'text': ' So very comprehensive across the United States.'}, {'timestamp': (563.0, 629.2), 'text': " And they were asking questions about the general cybersecurity environment, but in addition, some things came out around the workforce. So when those CISO's were asked about how they can support emerging threats and what their biggest challenges, number one, they said legacy infrastructure and solutions, number two, they said workforce. The number two thing, holding these state CISO's back from responding to the threats that are coming is the workforce. That's a pretty big issue. And the number two thing, the number two thing, the number two thing, holding these state CISO's back from responding to the threats that are coming is the workforce. That's a pretty big issue. And the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, the number two thing, holding these state-sea-sodes back from responding to the threats that are coming, is the workforce. That's a pretty big issue. In addition, 62% out-to-thirds reported that their staff have a gap in competencies. Don't have the skills to do the job. Another pretty big issue when you step back and look at it. When we talk about the belonging piece that Mara just talked about and shared, 23% with the respondents didn't know if their organization has established DEI leadership positions. So is there someone in the organization really focused on that inclusion piece and bringing all of your workforce along? And only 25% reported offering remote work options, which I know is tricky in the cyberspace. When you think about the sock and everything else, but it is there are ways to do it with at least parts of the workforce. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And we know that that's the same thing. And I know is tricky in the cyberspace. When you think about the sock and everything else, but it is there are ways to do it with at least parts of the workforce."}, {'timestamp': (629.2, 633.04), 'text': " And we know that that's super important to the millennials and slunder-reppers any groups."}, {'timestamp': (634.08, 638.4), 'text': ' Remote work has certainly been a topic since the pandemic, right? I see people nodding.'}, {'timestamp': (638.4, 641.76), 'text': " It's just on everybody's tongue, it's front of mind. So how do you make that work?"}, {'timestamp': (643.04, 706.76), 'text': " You know, addition to all of this, I think what's really important to know, right? Is that cyber threats have not produced in any way? So there's one stat that says that as a result of COVID-19, it's been linked to a 238% increase in cyber crime. So the numbers are going up the need continues and the resourcing and the workforce, maybe as a little bit lagging behind. In addition to this in a cyberspace, we're also seeing some pretty big macro trends in workforce space in general that are contributing to some of the challenges going on today. And these also probably won't surprise you. Number one, exhaustion. So this is where that burnout topic that some of you people are talking about is showing up. And in cyber agencies, we have frontline workers, right? There's a 24-7 operation. And so you're going to have this burnout challenge unfortunately. This is also contributed by working from home in some ways and that blurring of that work life home life when it ended the day happens if it happens and whose days on their laptop all night working. In addition, between 2021 and 2022, the number of meetings we all experience increased"}, {'timestamp': (706.76, 711.66), 'text': " 280 percent. So you're getting your Zoom fatigue, your Teams fatigue, right? Does that"}, {'timestamp': (711.66, 767.24), 'text': " meetings fatigue along the way? Resignation? Who's heard of the great resignation coming out of the pandemic? Yeah. So that's where the topic says it. We know that a lot of people left the workforce voluntarily and in fact in 2021 more people than ever before left the workforce according to the Bureau of Labor Statistics. Some people have come back and some people have it. So now our options for who's out there working and interested in being in the workforce and talent diversity in that they bring to your organizations, give you a little more limited. And then reshuffling. 34% of US workers shifted their cities since the start of COVID. 34% of the population is in the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of the state of US workers shifted their cities since the start of COVID. 34, a third of US workers have moved cities. And so this is where that remote work conversation becomes so important about can you get the right people. And so the people again that you have on board, how are you engaging them, keeping them, working in their most effective to support themselves in your organization?"}, {'timestamp': (767.24, 768.84), 'text': ' Why does all this matter?'}, {'timestamp': (768.84, 772.24), 'text': ' Because we know that losing employees is costly.'}, {'timestamp': (772.24, 775.6), 'text': ' Giving them a positive workforce experience means that they stay with your organization,'}, {'timestamp': (775.6, 782.4), 'text': ' they are committed, they are loyal, and they continue to grow and build within your organization.'}, {'timestamp': (782.4, 787.0), 'text': ' One study shows that losing a employee and having to recruit a new one,'}, {'timestamp': (787.0, 794.0), 'text': ' cost three to four times their salary. Yeah, opportunity costs the knowledge drain, the process'}, {'timestamp': (794.0, 801.0), 'text': ' of just onboarding the job up on the boards, bringing them on three to four times their salary.'}, {'timestamp': (801.0, 808.0), 'text': " So if you can get it right on the front end and really have that workforce experience be positive and keep your people it's going to pay off dividends in the actual."}, {'timestamp': (809.0, 812.0), 'text': " I'm going to pass it back to Mara who's going to tie this back to that model for us."}, {'timestamp': (813.0, 866.96), 'text': " I mean, building on what Susie said workforce experience has never been more important given this crisis and the urgency around keeping talented and finding the right talent. Since the workforce is an organization's host valuable asset, investing in workforce experience activities, we found because lead to more productive work, more efficient work, more innovative approaches to the work, and more engaged teams which ultimately results in better mission outcomes for your organization. And we found a direct correlation with the ability to retain employees, improve efficiency overall in your organization and the organization changes and transitions more effectively. From our work with other agencies as well as commercial organizations, we found that"}, {'timestamp': (866.96, 874.24), 'text': ' investing in work more experienced is the most sustainable way to create a competitive advantage'}, {'timestamp': (874.24, 879.84), 'text': " that improves the business and talent outcomes. And so, it's investing in work more"}, {'timestamp': (879.84, 914.0), 'text': " experience is a single best way that organizations can minimize workforce disruption in the short term. I'm going to get ahead of future work. How do we do that? And so in this next section we're going to focus on how cyber employers can win at workforce experience. So based on a 2022 state CIO, mass CO study that included 51 participants, it was identified that the following areas were some of the top priorities around how to improve workforce experience."}, {'timestamp': (927.52, 934.68), 'text': ' work options as particularly important 8% focus on flexible work schedules and how work affects their life 35% focus on how I grow as a human so focusing on'}, {'timestamp': (934.68, 941.56), 'text': ' reskilling opportunities for rotation programs and through the right training to make'}, {'timestamp': (941.56, 966.44), 'text': " sure that the workforce is able to meet the needs of the modern IT to man's. And so with this, Suzy is going to walk us through a deeper dive of the different options on how to address these results with this actually might look like in practice. Great. Thank you, Mara. So again, just to take it from the top, right? You know that the talent pipeline is tough. We know that we're recruiting processes challenging."}, {'timestamp': (966.44, 968.64), 'text': ' I think a little deeper into some of these things.'}, {'timestamp': (968.64, 970.96), 'text': " We're going to walk through a couple of activities, programs,"}, {'timestamp': (970.96, 974.48), 'text': ' initiatives, ideas, just to hopefully give you something to walk away'}, {'timestamp': (974.48, 977.8), 'text': ' with today that you can take back to your organization and implement.'}, {'timestamp': (977.8, 980.88), 'text': ' And the idea behind these is you can implement them tomorrow.'}, {'timestamp': (980.88, 984.44), 'text': " You don't need to get into an IT backlog of some system build."}, {'timestamp': (984.44, 1049.0), 'text': " You don't need to go to HR and say, I need, you know, millions of dollars to hire people and fill the gaps. These are things that you can do with your current workforce in place today to improve that workforce experience. So, starting with how I grow with a human and again 35% of the CIOs said that this was one of the most important things. A couple of ideas here for things that can be done and things that we've seen these accessible elsewhere. Number one, integrating training into the day-to-day. This is not about sending it employee off to go take a training, get a certification, step out of the workplace, and come back. We know that most adult learners prefer experiential learning. And so the way to do that is a apprenticeship, mentorship, shadowing, rotational programs, lunch and learns, informal training opportunities, action, after action reports where you can also around and talk about what did we do well, what didn't we do, what are we going to do differently. And some of this also is just documenting what it is you do. So setting up some of those SOPs so that you can pass those over to somebody and say, well this is the best way to do it. Here's a lesson's learned."}, {'timestamp': (1049.0, 1061.0), 'text': " Identifying opportunities for employees to use their strengths daily. When we talk about strengths, we don't just mean what am I good at. What we mean by strengths is what gives me individually energy."}, {'timestamp': (1061.0, 1072.0), 'text': " What energizes me? What excites me? And so I may be very good at data analytics, but that doesn't mean that that's my passion. I may actually prefer competing teams or communications or something like that."}, {'timestamp': (1072.0, 1079.0), 'text': ' And so checking in with your teams about what is it that they each person enjoys most and trying to align them to that work.'}, {'timestamp': (1079.0, 1107.0), 'text': " Studies have shown that if you align people to their strengths, again, the things that energize them, they will actually outperform, and do way better than if you give them a challenge task or activity that then they have to sort of overcome and be able to accomplish. We are good when we're playing to our strengths. So it doesn't mean that in a play you can only do their strength all day and be happy, but try to find ways for them to incorporate their strengths into the day to day as much as possible."}, {'timestamp': (1107.0, 1113.0), 'text': ' And then another thing that tends to be useful is stepping back and building looking at career pathways.'}, {'timestamp': (1113.0, 1119.0), 'text': ' So how are you going to grow people from level to level, from ability to ability?'}, {'timestamp': (1119.0, 1121.0), 'text': ' Is anybody here familiar with the nice framework?'}, {'timestamp': (1121.0, 1122.0), 'text': ' Come on, Nick.'}, {'timestamp': (1122.0, 1124.0), 'text': ' Yeah, a couple of hands.'}, {'timestamp': (1124.0, 1125.6), 'text': ' So this is a great resource'}, {'timestamp': (1125.6, 1130.96), 'text': " if you haven't seen an highly recommend going onto the website. They have work titles, they have"}, {'timestamp': (1130.96, 1136.96), 'text': ' job descriptions, they have competencies, they have skills, knowledge, experiences, and you can stack'}, {'timestamp': (1136.96, 1142.4), 'text': " roles so you can say, okay, if I'm an IT leadership management position today, and I want to become a data"}, {'timestamp': (1142.4, 1207.0), 'text': " analyst, and they will help you kind of, they map that path a little bit for you with the overlap between those two roles for example and then some of the case this knowledge skills and abilities to go get from one to the other. So a very very useful tool used by a lot of cyber agencies and organizations highly highly highly highly recommend using it or at least checking an out and seeing if there's anything useful for you there. But really looking at where are my people going and how to like grow them because we know also that one of the most important things that we have to do is to make sure that we have a lot of information about the data that we have to do with the data and that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that we have to do with the data that're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it. And we know that we're doing it environment. I know the sock is what it is. We know that piece. But there are conversations still to be having around work location. What we have found is most successful is when organizations really focus in on work location. In alignment to your desires, your goals, your strategy. There are some things that working remote are great for."}, {'timestamp': (1207.0, 1211.0), 'text': ' Heds downtime, idea generation, focus, and productivity.'}, {'timestamp': (1211.0, 1216.0), 'text': ' There are some things that working in person are great for.'}, {'timestamp': (1216.0, 1220.0), 'text': ' Teamwork, apprenticeship, and some of those sort of incidental connection points,'}, {'timestamp': (1220.0, 1230.0), 'text': " running into somebody, you know, by the water cooler and finding out that we're doing similar things, and we can train notess actually Mara and I just did that and I said oh send me that if you could because I want to see that right?"}, {'timestamp': (1230.0, 1238.0), 'text': " So designing intentionally for what work can be done in person, what work can be done remotely, maybe it's part and part."}, {'timestamp': (1238.0, 1266.6), 'text': " Maybe there's a single initiative or project where you can send people to go home and work remotely and then come back and they're still feeling like they're getting a taste of that opportunity as they're normally meeting to be on site. So really being smart and intentional and purposeful about how you design that. Looking at the digital communications and information sharing. So a lot of us have moved on to teams and Slack and Zoom and these digital tools. Are you really using them for their maximum capability to support your organization? Is there"}, {'timestamp': (1266.6, 1271.6), 'text': ' somebody in your organization who really is tasked with understanding all of the features that you have'}, {'timestamp': (1271.6, 1278.8), 'text': ' license to and bringing that goodness to your organization to help your people work even better? And then the third one there is forced'}, {'timestamp': (1278.8, 1285.6), 'text': ' to connectivity and networking especially in a hybrid or remote environment. We know that happiness at work and elsewhere'}, {'timestamp': (1285.6, 1287.4), 'text': ' is tied to social connection.'}, {'timestamp': (1287.4, 1290.72), 'text': ' So how are you forcing some of that social connection?'}, {'timestamp': (1290.72, 1292.6), 'text': ' Maybe it is in those brown bag trainings'}, {'timestamp': (1292.6, 1294.12), 'text': ' or after action reports.'}, {'timestamp': (1294.12, 1296.56), 'text': ' Maybe it is taking a moment before diving into a meeting'}, {'timestamp': (1296.56, 1298.32), 'text': " and saying, hey, how is everybody's weekend?"}, {'timestamp': (1298.32, 1299.8), 'text': ' Or what are you doing this weekend?'}, {'timestamp': (1299.8, 1301.04), 'text': ' Or tell me something funny?'}, {'timestamp': (1301.04, 1304.64), 'text': ' Or if you were going to be a office supply, what would you be?'}, {'timestamp': (1304.64, 1361.0), 'text': " Whatever it may be, right? But finding a way to sort of force a little door opening and having people build that connection. And then the well being piece. And so I shared with you about the exhaustion, you know, and the burnout rates and all that earlier. A lot of organizations, especially since during and since the pandemic, have stood up well being activities. They're not necessarily aligned to a situation where they're not going to be able to do that. And so I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to be able to do that. And I'm going to do that. And I'm going to be able to do that. And I'm going to do that. And I'm going to do that. And I'm going to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that it has a lot of opportunities to do that. And then remember that. So it's not just sort of a scatter plot of activities, but there's sort of intentional and organized aligning back to those goals that you have. And then, remember that impact of that and adjust as needed. And if you're engaging your workforce correctly, then you'll have lots of data points about what's working and what's not in order to make those adjustments."}, {'timestamp': (1385.0, 1527.6), 'text': " make those adjustments. Started on this a little bit, but again, thinking creatively, not just about remote work, but how shift work is scheduled. So a lot of the conversation around workforce and economic development is around people who may be to be taking their child care duties at home while still trying to get into the workforce. What is it that you can do to maybe look at four hour shifts or six hour shifts different than your typical shifts to bring in more people in diversity, and also alleviate some of the burnout and pressure that's happening for the workforce in place today. There are a bunch of apps that focus on this. A lot of them are focusing in the health care space of the transportation frontline workers space, but they could easily be leveraged over into the cyberspace as well. And then the third one here conducting a culture assessment. A lot of organizations assume that culture sort of just happens and it doesn't. It's important to be intentional about culture and what do we mean when we talk about culture, right? Because we all sort of live in it all the time, but what is that? When we talk about culture, what we talk about are the intangibles, your beliefs and your values as an organization. Are we more individually individualistic, the ability to talk about culture, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about, what we talk about culture, what we talk about are the intangibles, your beliefs and your values as an organization. Are we more individually individualistic, focused or are we more collaborative? Are we more risk of first? Or are we more innovative and risk takers? Are we more internally focused or do we have external customers that we've really organized around? And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be and what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be. And what are the steps to get from a to the other. And so when you do a cultural assessment, you look at where you are today versus where you want your organization to be. And what are the steps to get from a to the other. And so when you do a, what are the steps to get from a to the other. And so when you do a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, what are the steps to get from a, where you want your organization to be, and what are the steps to get from A to Z? And how do you do that in a way that then filters and flows through your entire organization? So that by the time you get to performance reviews or your next strategic planning cycle, we're standing up a new program or redesigning your organization, there is an friction with what it is you really want from your employees and what you're trying to achieve as an organization and how you're getting there. So cultural assessment, well, it seems like a sort of unnecessary. It's actually very effective and super supportive of your articles. Okay, I've talked a lot. Lots of ideas. Hopefully, good menu for you to pick at least something from a fithesical. I'm going to pass it back to Mara to kind of bring it all together for us. Thank you, Sissy. So, how can employers win at work for six variants? So, practically speaking, they keep it implementing a successful work for six variants. Really starts with understanding what your workforce wants. That can be through a survey or focus groups doing doing user research, but really putting their voice at the center of these initiatives. And then working to prioritize which ones within your organization and help you achieve your overall business objectives. So we're going to walk through our perspective on what it means to really bring work for"}, {'timestamp': (1527.6, 1535.04), 'text': ' six periods to an organization. So starting with employers need to think about beyond employment engagement.'}, {'timestamp': (1535.04, 1541.28), 'text': " And so what we mean by this is it's not just one data point, our employees engaged, you know,"}, {'timestamp': (1541.28, 1567.04), 'text': " how I retained my staffer did I recruit an attractive, you know, this number of individuals? It's beyond that and a combination of those aid characteristics that really helps to retain employees and attract the right talent and enhance the overall workforce experience. So an example of that is, you know, looking at physical workspace. So, you know, do they are they able to do their work remotely? In some cases,"}, {'timestamp': (1567.04, 1572.08), 'text': ' as Susie mentioned, they do they have everything they need in the office in order to be productive,'}, {'timestamp': (1572.08, 1578.24), 'text': ' to get things done in terms of head-stallam work versus hue collaboration, culture, do they,'}, {'timestamp': (1578.24, 1614.0), 'text': " are they recognized for their work? It was great as a hundred dollar gift card is for a great job and something. It in a lot of cases what we found is it's equally just as advantageous to how we manage our reach out and saying thank you so much. You did a great job and acknowledge that. And that's why we know what can motivate and keep employees engaged. And another area is technology. I think we mentioned you know it's a way to help collaborate, work remotely, but it can also be an impediment to when technology isn't easily accessible within organization."}, {'timestamp': (1614.0, 1635.0), 'text': ' There are challenges, IT is kind of available to help with certain things, but how would you approach across those eight dimensions multiple areas of prioritizing different initiatives?'}, {'timestamp': (1635.0, 1647.6), 'text': " The second one that we really focus on and this goes back to its Susie set of Fiesta's and also what I mentioned around the voice of the workforce. And so, incorporating that"}, {'timestamp': (1647.6, 1653.28), 'text': ' throughout the design is incredibly important. One of the first things that we often do is pull'}, {'timestamp': (1653.28, 1659.2), 'text': ' survey of a particular group to understand what they really hear about. What are their preferences?'}, {'timestamp': (1659.2, 1675.0), 'text': " If they had, it's not just, I want more money, but but rather if you had a choice between a little bit more flexibility in your work schedule versus you know increased recognition, you know, where do they release stand in terms of what should actually get prioritized?"}, {'timestamp': (1675.0, 1728.8), 'text': " And when you drill down into the art of the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work for the work even, you know, communicating that you're doing this and engaging the workforce early on and incorporating them into this because this is important to the organization and to you as leaders is an important way. I think also to retain talent. The third one is a continuous listing approach and this one really focuses on not just pulsing"}, {'timestamp': (1729.28, 1735.6), 'text': ' a workforce once a year through an annual HR survey of how do you really feel like you know what'}, {'timestamp': (1736.16, 1742.48), 'text': ' leadership considerations should we implement or you know how can we enhance the performance management process?'}, {'timestamp': (1742.8, 1768.4), 'text': " It's really on an ongoing basis and even in in an informal way, you as leaders and managers of teams asking your teams, you know, what they care about, what their frustrated about, what their preferences are, and taking small steps where you can to try to incorporate that into how they work. And you know, sometimes it's challenging to do organization-wide changes that take a lot of funding and investment and capabilities. So often, you know, you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, you can do a lot of things. And you know, sometimes it's challenging to do organization why changes that take a lot of funding, investment,"}, {'timestamp': (1768.4, 1769.56), 'text': ' and capabilities.'}, {'timestamp': (1769.56, 1773.0), 'text': ' So oftentimes, a way to do that is through, you know,'}, {'timestamp': (1773.0, 1776.96), 'text': " more informal touch points on individual teams that you're leading."}, {'timestamp': (1776.96, 1780.76), 'text': " We found that that's a concept that's a quick way to be able"}, {'timestamp': (1780.76, 1784.08), 'text': ' to get a lot of employee engagement and routine'}, {'timestamp': (1784.08, 1787.84), 'text': ' and keep your talent happy. The final one is'}, {'timestamp': (1787.84, 1794.08), 'text': " workplace technology and we've talked about this a little bit and I think really with this it's"}, {'timestamp': (1794.08, 1800.72), 'text': " how do you boost productivity? Workers want to do a great job. They're they're at the perform well. They,"}, {'timestamp': (1800.72, 1810.0), 'text': ' you know, are often very mission-driven. They want to grow their careers and they want to try to be as efficient as they can in their job.'}, {'timestamp': (1810.0, 1818.0), 'text': " But oftentimes, the organization decides in terms of technology versus what the worker actually needs. There's a bit of a disconnect."}, {'timestamp': (1818.0, 1970.88), 'text': " And so really looking at how this can give you a major accelerator based on how you work. Do you want an hands collaboration? Do you want to try to create more opportunities for a hybrid or remote work? And, you know, through technology, that can really enhance and accelerate a lot of those activities. And so with all of these four areas combined, this is what we've seen across, you know, four-Jet 500 companies, federal agencies, city, city, agencies as well, as kind of the key characteristics and commonalities amongst the most effective and fully engagement and workforce experience programs. And just to kind of round things out with this globe is, you know, if workforce experience, I recognize, you know, can seem kind of fluffy and like, oh, that's a nice to have. But our premise is that your workforce is the most important asset in your organization. And if they are in the most important asset, you really want to invest in them. To, you know, day to day, be the driver of change in, you know, be the lower productivity. We've just found that, you know, by investing in this, you know, day to day be the driver of change in, you know, be the alert productivity. We've just found that, you know, by investing in this and putting the workforce as, you know, the center part of what you invest in as an organization in leaders. It's not only about retention talent, you know, the cyber workforce crisis, but people want to do work well and they're able to get more done and achieve more without you, you know, directly supervising and micro managing or looking at everything because they just genuinely want to work for you and are able to be more productive overall in that way. So with that, that is our presentation for today. I hope there's a little bit of, you know, the landscape of the cyber workforce with some practical tips that you can take away for how to just think about, you know, improving the overall workforce experience and investing in your employees. So with this, you know, we know that all of you are in the trenches every day. You're facing this, you're living this. And we are just interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you. And we are interested to hear from you more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going to be a little bit more excited. I'm going get someone. Sorry. Don't be shy. So yeah, I'm so"}, {'timestamp': (1970.88, 1975.04), 'text': " worried. How is something or what hasn't worked well, you know, that's something that's"}, {'timestamp': (1975.04, 1979.6), 'text': " easier to start with too. But we'd love to just hear from all of you because I think, you know,"}, {'timestamp': (1979.6, 1984.8), 'text': " how we've aggregated a lot of these best practices and what we've come with, it is by, you know,"}, {'timestamp': (1984.8, None), 'text': " hearing other experiences from other organizations. And so, one of the best and most effective things, I feel like I take away from conferences often is hearing from many peers and what they're facing and, you know, let, you know, similarly, I could bring back to what I'm leading. What about you? Oh, I see him back there. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I'm going to the water. I Thank you."}]} \ No newline at end of file diff --git a/utils.py b/utils.py new file mode 100644 index 00000000..b15f992b --- /dev/null +++ b/utils.py @@ -0,0 +1,7 @@ +import asyncio +from functools import partial + +def run_in_executor(func, *args, executor=None, **kwargs): + callback = partial(func, *args, **kwargs) + loop = asyncio.get_event_loop() + return asyncio.get_event_loop().run_in_executor(executor, callback)