Skip to content

serve.gradio_user_interface_base

GradioUserInference

Source code in src/python/easydel/serve/gradio_user_interface_base.py 
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
class GradioUserInference:
    @staticmethod
    def chat_interface_components(
            sample_func: typing.Callable,
            max_sequence_length: int,
            max_new_tokens: int,
            max_compile_tokens: int
    ):
        """
        The function `chat_interface_components` creates the components for a chat interface, including
        a chat history, message box, buttons for submitting, stopping, and clearing the conversation,
        and sliders for advanced options.
        """

        _max_length = max_sequence_length
        _max_new_tokens = max_new_tokens
        _max_compile_tokens = max_compile_tokens

        with gr.Column("100%"):
            gr.Markdown(
                "# <h1><center style='color:white;'>Powered by "
                "[EasyDeL](https://github.com/erfanzar/EasyDeL)</center></h1>",
            )
            history = gr.Chatbot(
                elem_id="easydel",
                label="easydel",
                container=True,
                height="65vh",
            )
            prompt = gr.Textbox(
                show_label=False, placeholder='Enter Your Prompt Here.', container=False
            )
            with gr.Row():
                submit = gr.Button(
                    value="Run",
                    variant="primary"
                )
                stop = gr.Button(
                    value='Stop'
                )
                clear = gr.Button(
                    value='Clear Conversation'
                )
            with gr.Accordion(open=False, label="Advanced Options"):
                system_prompt = gr.Textbox(
                    value="",
                    show_label=False,
                    label="System Prompt",
                    placeholder='System Prompt',
                    container=False
                )

                max_sequence_length = gr.Slider(
                    value=_max_length,
                    maximum=10000,
                    minimum=1,
                    label='Max Tokens',
                    step=1
                )

                max_new_tokens = gr.Slider(
                    value=_max_new_tokens,
                    maximum=10000,
                    minimum=_max_compile_tokens,
                    label='Max New Tokens',
                    step=_max_compile_tokens
                )

                max_compile_tokens = gr.Slider(
                    value=_max_compile_tokens,
                    maximum=_max_compile_tokens,
                    minimum=_max_compile_tokens,
                    label='Max Compile Tokens',
                    step=_max_compile_tokens
                )

                temperature = gr.Slider(
                    value=0.8,
                    maximum=1,
                    minimum=0.1,
                    label='Temperature',
                    step=0.01
                )
                top_p = gr.Slider(
                    value=0.9,
                    maximum=1,
                    minimum=0.1,
                    label='Top P',
                    step=0.01
                )
                top_k = gr.Slider(
                    value=50,
                    maximum=100,
                    minimum=1,
                    label='Top K',
                    step=1
                )
                repetition_penalty = gr.Slider(
                    value=1.2,
                    maximum=5,
                    minimum=0.1,
                    label='Repetition Penalty'
                )
                greedy = gr.Radio(
                    value=True,
                    label="Do Sample or Greedy Generation"
                )

                mode = gr.Dropdown(
                    choices=["Chat", "Instruct"],
                    value="Chat",
                    label="Mode",
                    multiselect=False
                )

        inputs = [
            prompt,
            history,
            system_prompt,
            mode,
            max_sequence_length,
            max_new_tokens,
            max_compile_tokens,
            greedy,
            temperature,
            top_p,
            top_k,
            repetition_penalty
        ]

        clear.click(fn=lambda: [], outputs=[history])
        sub_event = submit.click(
            fn=sample_func, inputs=inputs, outputs=[prompt, history]
        )
        txt_event = prompt.submit(
            fn=sample_func, inputs=inputs, outputs=[prompt, history]
        )
        stop.click(
            fn=None,
            inputs=None,
            outputs=None,
            cancels=[txt_event, sub_event]
        )

    def sample_gradio(
            self,
            prompt: str,
            history: List[List[str]],
            system_prompt: typing.Optional[str],
            mode: str,
            max_sequence_length: int,
            max_new_tokens: int,
            max_compile_tokens: int,
            greedy: bool,
            temperature: float,
            top_p: float,
            top_k: int,
            repetition_penalty: float
    ):
        raise NotImplementedError()

    def build_inference(
            self,
            sample_func: typing.Callable,
            max_sequence_length: int,
            max_new_tokens: int,
            max_compile_tokens: int
    ) -> gr.Blocks:
        """
        The function "build_inference" returns a gr.Blocks object that model
        interface components.
        :return: a gr.Blocks object.
        """
        with gr.Blocks(
                theme=seafoam
        ) as block:
            self.chat_interface_components(
                sample_func=sample_func,
                max_sequence_length=max_sequence_length,
                max_new_tokens=max_new_tokens,
                max_compile_tokens=max_compile_tokens
            )
        return block

    def __repr__(self):

        """
        The __repr__ function is used to generate a string representation of an object.
        This function should return a string that can be parsed by the Python interpreter
        to recreate the object. The __repr__ function is called when you use print() on an
        object, or when you type its name in the REPL.

        :param self: Refer to the instance of the class
        :return: A string representation of the object
        """
        string = f"{self.__class__.__name__}(\n"
        for k, v in self.__dict__.items():
            if not k.startswith("_"):

                try:
                    repr_src = f"\t{k} : " + v.__str__().replace("\n", "\n\t") + "\n"
                    string += repr_src if len(repr_src) < 500 else f"\t{k} : " + f"{v.__class__.__name__}(...)" + "\n"
                except TypeError:
                    ...

        return string + ")"

    def __str__(self):

        """
        The __str__ function is called when you use the print function or when str() is used.
        It should return a string representation of the object.

        :param self: Refer to the instance of the class
        :return: The object's string representation
        """
        return self.__repr__()

__repr__()

The repr function is used to generate a string representation of an object. This function should return a string that can be parsed by the Python interpreter to recreate the object. The repr function is called when you use print() on an object, or when you type its name in the REPL.

Parameters:

Name Type Description Default
self

Refer to the instance of the class

 required

Returns:

Type Description

A string representation of the object
Source code in src/python/easydel/serve/gradio_user_interface_base.py 
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def __repr__(self):

    """
    The __repr__ function is used to generate a string representation of an object.
    This function should return a string that can be parsed by the Python interpreter
    to recreate the object. The __repr__ function is called when you use print() on an
    object, or when you type its name in the REPL.

    :param self: Refer to the instance of the class
    :return: A string representation of the object
    """
    string = f"{self.__class__.__name__}(\n"
    for k, v in self.__dict__.items():
        if not k.startswith("_"):

            try:
                repr_src = f"\t{k} : " + v.__str__().replace("\n", "\n\t") + "\n"
                string += repr_src if len(repr_src) < 500 else f"\t{k} : " + f"{v.__class__.__name__}(...)" + "\n"
            except TypeError:
                ...

    return string + ")"

__str__()

The str function is called when you use the print function or when str() is used. It should return a string representation of the object.

Parameters:

Name Type Description Default
self

Refer to the instance of the class

 required

Returns:

Type Description

The object's string representation
Source code in src/python/easydel/serve/gradio_user_interface_base.py 
215
216
217
218
219
220
221
222
223
224
def __str__(self):

    """
    The __str__ function is called when you use the print function or when str() is used.
    It should return a string representation of the object.

    :param self: Refer to the instance of the class
    :return: The object's string representation
    """
    return self.__repr__()

build_inference(sample_func, max_sequence_length, max_new_tokens, max_compile_tokens)

The function "build_inference" returns a gr.Blocks object that model interface components.

Returns:

Type Description
Blocks

a gr.Blocks object.
Source code in src/python/easydel/serve/gradio_user_interface_base.py 
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def build_inference(
        self,
        sample_func: typing.Callable,
        max_sequence_length: int,
        max_new_tokens: int,
        max_compile_tokens: int
) -> gr.Blocks:
    """
    The function "build_inference" returns a gr.Blocks object that model
    interface components.
    :return: a gr.Blocks object.
    """
    with gr.Blocks(
            theme=seafoam
    ) as block:
        self.chat_interface_components(
            sample_func=sample_func,
            max_sequence_length=max_sequence_length,
            max_new_tokens=max_new_tokens,
            max_compile_tokens=max_compile_tokens
        )
    return block

chat_interface_components(sample_func, max_sequence_length, max_new_tokens, max_compile_tokens) staticmethod

The function chat_interface_components creates the components for a chat interface, including a chat history, message box, buttons for submitting, stopping, and clearing the conversation, and sliders for advanced options.

Source code in src/python/easydel/serve/gradio_user_interface_base.py 
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
@staticmethod
def chat_interface_components(
        sample_func: typing.Callable,
        max_sequence_length: int,
        max_new_tokens: int,
        max_compile_tokens: int
):
    """
    The function `chat_interface_components` creates the components for a chat interface, including
    a chat history, message box, buttons for submitting, stopping, and clearing the conversation,
    and sliders for advanced options.
    """

    _max_length = max_sequence_length
    _max_new_tokens = max_new_tokens
    _max_compile_tokens = max_compile_tokens

    with gr.Column("100%"):
        gr.Markdown(
            "# <h1><center style='color:white;'>Powered by "
            "[EasyDeL](https://github.com/erfanzar/EasyDeL)</center></h1>",
        )
        history = gr.Chatbot(
            elem_id="easydel",
            label="easydel",
            container=True,
            height="65vh",
        )
        prompt = gr.Textbox(
            show_label=False, placeholder='Enter Your Prompt Here.', container=False
        )
        with gr.Row():
            submit = gr.Button(
                value="Run",
                variant="primary"
            )
            stop = gr.Button(
                value='Stop'
            )
            clear = gr.Button(
                value='Clear Conversation'
            )
        with gr.Accordion(open=False, label="Advanced Options"):
            system_prompt = gr.Textbox(
                value="",
                show_label=False,
                label="System Prompt",
                placeholder='System Prompt',
                container=False
            )

            max_sequence_length = gr.Slider(
                value=_max_length,
                maximum=10000,
                minimum=1,
                label='Max Tokens',
                step=1
            )

            max_new_tokens = gr.Slider(
                value=_max_new_tokens,
                maximum=10000,
                minimum=_max_compile_tokens,
                label='Max New Tokens',
                step=_max_compile_tokens
            )

            max_compile_tokens = gr.Slider(
                value=_max_compile_tokens,
                maximum=_max_compile_tokens,
                minimum=_max_compile_tokens,
                label='Max Compile Tokens',
                step=_max_compile_tokens
            )

            temperature = gr.Slider(
                value=0.8,
                maximum=1,
                minimum=0.1,
                label='Temperature',
                step=0.01
            )
            top_p = gr.Slider(
                value=0.9,
                maximum=1,
                minimum=0.1,
                label='Top P',
                step=0.01
            )
            top_k = gr.Slider(
                value=50,
                maximum=100,
                minimum=1,
                label='Top K',
                step=1
            )
            repetition_penalty = gr.Slider(
                value=1.2,
                maximum=5,
                minimum=0.1,
                label='Repetition Penalty'
            )
            greedy = gr.Radio(
                value=True,
                label="Do Sample or Greedy Generation"
            )

            mode = gr.Dropdown(
                choices=["Chat", "Instruct"],
                value="Chat",
                label="Mode",
                multiselect=False
            )

    inputs = [
        prompt,
        history,
        system_prompt,
        mode,
        max_sequence_length,
        max_new_tokens,
        max_compile_tokens,
        greedy,
        temperature,
        top_p,
        top_k,
        repetition_penalty
    ]

    clear.click(fn=lambda: [], outputs=[history])
    sub_event = submit.click(
        fn=sample_func, inputs=inputs, outputs=[prompt, history]
    )
    txt_event = prompt.submit(
        fn=sample_func, inputs=inputs, outputs=[prompt, history]
    )
    stop.click(
        fn=None,
        inputs=None,
        outputs=None,
        cancels=[txt_event, sub_event]
    )