| 1 | { |
|---|
| 2 | "nbformat": 4, |
|---|
| 3 | "nbformat_minor": 0, |
|---|
| 4 | "metadata": { |
|---|
| 5 | "kernelspec": { |
|---|
| 6 | "display_name": "Python 3", |
|---|
| 7 | "language": "python", |
|---|
| 8 | "name": "python3" |
|---|
| 9 | }, |
|---|
| 10 | "language_info": { |
|---|
| 11 | "codemirror_mode": { |
|---|
| 12 | "name": "ipython", |
|---|
| 13 | "version": 3 |
|---|
| 14 | }, |
|---|
| 15 | "file_extension": ".py", |
|---|
| 16 | "mimetype": "text/x-python", |
|---|
| 17 | "name": "python", |
|---|
| 18 | "nbconvert_exporter": "python", |
|---|
| 19 | "pygments_lexer": "ipython3", |
|---|
| 20 | "version": "3.6.8" |
|---|
| 21 | }, |
|---|
| 22 | "nbpresent": { |
|---|
| 23 | "slides": { |
|---|
| 24 | "004b13a1-f767-463b-80f7-8599c4e09e5a": { |
|---|
| 25 | "id": "004b13a1-f767-463b-80f7-8599c4e09e5a", |
|---|
| 26 | "prev": "af9260f5-c3c0-4457-8946-fc08623fdff8", |
|---|
| 27 | "regions": { |
|---|
| 28 | "a6fcc3e3-cb1a-44bf-b8c6-342f5fe53056": { |
|---|
| 29 | "attrs": { |
|---|
| 30 | "height": 0.8, |
|---|
| 31 | "width": 0.8, |
|---|
| 32 | "x": 0.1, |
|---|
| 33 | "y": 0.1 |
|---|
| 34 | }, |
|---|
| 35 | "content": { |
|---|
| 36 | "cell": "5c48f967-8d12-43d2-87e8-a08e5ebfb660", |
|---|
| 37 | "part": "whole" |
|---|
| 38 | }, |
|---|
| 39 | "id": "a6fcc3e3-cb1a-44bf-b8c6-342f5fe53056" |
|---|
| 40 | } |
|---|
| 41 | } |
|---|
| 42 | }, |
|---|
| 43 | "09c98d86-d894-4fd9-847e-1b89789b4e7f": { |
|---|
| 44 | "id": "09c98d86-d894-4fd9-847e-1b89789b4e7f", |
|---|
| 45 | "prev": null, |
|---|
| 46 | "regions": { |
|---|
| 47 | "2256868b-7797-45fd-8b58-fc7f5a861635": { |
|---|
| 48 | "attrs": { |
|---|
| 49 | "height": 0.8, |
|---|
| 50 | "width": 0.8, |
|---|
| 51 | "x": 0.1, |
|---|
| 52 | "y": 0.1 |
|---|
| 53 | }, |
|---|
| 54 | "content": { |
|---|
| 55 | "cell": "f3dbef4d-9bf4-4e24-bcd0-a4ce3dd00fb8", |
|---|
| 56 | "part": "whole" |
|---|
| 57 | }, |
|---|
| 58 | "id": "2256868b-7797-45fd-8b58-fc7f5a861635" |
|---|
| 59 | } |
|---|
| 60 | } |
|---|
| 61 | }, |
|---|
| 62 | "190227c4-a075-4074-ae84-e9ddf8e69560": { |
|---|
| 63 | "id": "190227c4-a075-4074-ae84-e9ddf8e69560", |
|---|
| 64 | "prev": "9726092c-f997-4a46-bef8-d0ea1cc64a60", |
|---|
| 65 | "regions": { |
|---|
| 66 | "68b26cc7-174a-4407-aea6-8930a804b4ec": { |
|---|
| 67 | "attrs": { |
|---|
| 68 | "height": 0.8, |
|---|
| 69 | "width": 0.8, |
|---|
| 70 | "x": 0.1, |
|---|
| 71 | "y": 0.1 |
|---|
| 72 | }, |
|---|
| 73 | "content": { |
|---|
| 74 | "cell": "9874742a-f8f7-4cf9-8f3c-a040b47c41d9", |
|---|
| 75 | "part": "whole" |
|---|
| 76 | }, |
|---|
| 77 | "id": "68b26cc7-174a-4407-aea6-8930a804b4ec" |
|---|
| 78 | } |
|---|
| 79 | } |
|---|
| 80 | }, |
|---|
| 81 | "1b6bbadb-1200-45d3-a5e8-717091642c54": { |
|---|
| 82 | "id": "1b6bbadb-1200-45d3-a5e8-717091642c54", |
|---|
| 83 | "prev": "dab15b59-64b4-4c70-b208-dfdc24a948aa", |
|---|
| 84 | "regions": { |
|---|
| 85 | "7ce7bd82-d3d4-4803-8dd9-528da5d2c270": { |
|---|
| 86 | "attrs": { |
|---|
| 87 | "height": 0.8, |
|---|
| 88 | "width": 0.8, |
|---|
| 89 | "x": 0.1, |
|---|
| 90 | "y": 0.1 |
|---|
| 91 | }, |
|---|
| 92 | "content": { |
|---|
| 93 | "cell": "60ff129a-80bd-4c96-8f5e-8c2284f71f1c", |
|---|
| 94 | "part": "whole" |
|---|
| 95 | }, |
|---|
| 96 | "id": "7ce7bd82-d3d4-4803-8dd9-528da5d2c270" |
|---|
| 97 | } |
|---|
| 98 | } |
|---|
| 99 | }, |
|---|
| 100 | "1ba27192-2b68-45b9-96f2-381167e1f808": { |
|---|
| 101 | "id": "1ba27192-2b68-45b9-96f2-381167e1f808", |
|---|
| 102 | "prev": "8f08e261-fb4a-4f9e-a021-23e6542c4a69", |
|---|
| 103 | "regions": { |
|---|
| 104 | "f20f9b58-448a-4e09-8a7a-e557b96affff": { |
|---|
| 105 | "attrs": { |
|---|
| 106 | "height": 0.8, |
|---|
| 107 | "width": 0.8, |
|---|
| 108 | "x": 0.1, |
|---|
| 109 | "y": 0.1 |
|---|
| 110 | }, |
|---|
| 111 | "content": { |
|---|
| 112 | "cell": "9dda77d1-4b69-48c2-b603-750748d4af25", |
|---|
| 113 | "part": "whole" |
|---|
| 114 | }, |
|---|
| 115 | "id": "f20f9b58-448a-4e09-8a7a-e557b96affff" |
|---|
| 116 | } |
|---|
| 117 | } |
|---|
| 118 | }, |
|---|
| 119 | "24da9cf4-1ecd-4314-a35e-fa3ed8056970": { |
|---|
| 120 | "id": "24da9cf4-1ecd-4314-a35e-fa3ed8056970", |
|---|
| 121 | "prev": "6a61d64d-530f-4f94-a29e-7bdf66bfaadb", |
|---|
| 122 | "regions": { |
|---|
| 123 | "c935fde2-6e13-4ad3-a10d-107d6cd77246": { |
|---|
| 124 | "attrs": { |
|---|
| 125 | "height": 0.8, |
|---|
| 126 | "width": 0.8, |
|---|
| 127 | "x": 0.1, |
|---|
| 128 | "y": 0.1 |
|---|
| 129 | }, |
|---|
| 130 | "content": { |
|---|
| 131 | "cell": "f06795b7-b03f-4954-9ca6-3a1fb19fa500", |
|---|
| 132 | "part": "whole" |
|---|
| 133 | }, |
|---|
| 134 | "id": "c935fde2-6e13-4ad3-a10d-107d6cd77246" |
|---|
| 135 | } |
|---|
| 136 | } |
|---|
| 137 | }, |
|---|
| 138 | "2c78a52b-56af-4232-ab1b-16abbe0a79bc": { |
|---|
| 139 | "id": "2c78a52b-56af-4232-ab1b-16abbe0a79bc", |
|---|
| 140 | "prev": "eb9a3413-8bc3-4e0b-b188-ef05e5f0c461", |
|---|
| 141 | "regions": { |
|---|
| 142 | "5e590248-29f7-48a3-a6f3-55d4b67755a6": { |
|---|
| 143 | "attrs": { |
|---|
| 144 | "height": 0.8, |
|---|
| 145 | "width": 0.8, |
|---|
| 146 | "x": 0.1, |
|---|
| 147 | "y": 0.1 |
|---|
| 148 | }, |
|---|
| 149 | "content": { |
|---|
| 150 | "cell": "74d938a4-4600-41ae-8cab-dd359e64cfa5", |
|---|
| 151 | "part": "whole" |
|---|
| 152 | }, |
|---|
| 153 | "id": "5e590248-29f7-48a3-a6f3-55d4b67755a6" |
|---|
| 154 | } |
|---|
| 155 | } |
|---|
| 156 | }, |
|---|
| 157 | "375c8fec-6f3b-4b01-89ad-410c4f27a4d0": { |
|---|
| 158 | "id": "375c8fec-6f3b-4b01-89ad-410c4f27a4d0", |
|---|
| 159 | "prev": "c24ac2b3-9975-4ee4-b59c-df7f94db8722", |
|---|
| 160 | "regions": { |
|---|
| 161 | "7a6d15da-c3bd-480c-9dcf-52f2a1805a2c": { |
|---|
| 162 | "attrs": { |
|---|
| 163 | "height": 0.8, |
|---|
| 164 | "width": 0.8, |
|---|
| 165 | "x": 0.1, |
|---|
| 166 | "y": 0.1 |
|---|
| 167 | }, |
|---|
| 168 | "content": { |
|---|
| 169 | "cell": "61acf6bd-aeec-49fb-874a-ec19691ee757", |
|---|
| 170 | "part": "whole" |
|---|
| 171 | }, |
|---|
| 172 | "id": "7a6d15da-c3bd-480c-9dcf-52f2a1805a2c" |
|---|
| 173 | } |
|---|
| 174 | } |
|---|
| 175 | }, |
|---|
| 176 | "3feede8b-0bbd-42b9-915c-56e40727cf62": { |
|---|
| 177 | "id": "3feede8b-0bbd-42b9-915c-56e40727cf62", |
|---|
| 178 | "prev": "f066aa28-05ba-4dde-81e2-7a9b7f104c55", |
|---|
| 179 | "regions": { |
|---|
| 180 | "35b0bd5b-37e5-4d53-95b0-d70d8d1ba55c": { |
|---|
| 181 | "attrs": { |
|---|
| 182 | "height": 0.8, |
|---|
| 183 | "width": 0.8, |
|---|
| 184 | "x": 0.1, |
|---|
| 185 | "y": 0.1 |
|---|
| 186 | }, |
|---|
| 187 | "content": { |
|---|
| 188 | "cell": "d0a48142-bed2-48de-a84b-219d3139581f", |
|---|
| 189 | "part": "whole" |
|---|
| 190 | }, |
|---|
| 191 | "id": "35b0bd5b-37e5-4d53-95b0-d70d8d1ba55c" |
|---|
| 192 | } |
|---|
| 193 | } |
|---|
| 194 | }, |
|---|
| 195 | "49ff324a-a018-43ac-92a1-caeefeda1d0b": { |
|---|
| 196 | "id": "49ff324a-a018-43ac-92a1-caeefeda1d0b", |
|---|
| 197 | "prev": "e81dcf87-fb95-4d33-89f9-6d820212ff9e", |
|---|
| 198 | "regions": { |
|---|
| 199 | "52348175-fe80-4346-9fe7-e76b63ac6c47": { |
|---|
| 200 | "attrs": { |
|---|
| 201 | "height": 0.8, |
|---|
| 202 | "width": 0.8, |
|---|
| 203 | "x": 0.1, |
|---|
| 204 | "y": 0.1 |
|---|
| 205 | }, |
|---|
| 206 | "content": { |
|---|
| 207 | "cell": "332417aa-251e-4477-bac7-59e34406c66e", |
|---|
| 208 | "part": "whole" |
|---|
| 209 | }, |
|---|
| 210 | "id": "52348175-fe80-4346-9fe7-e76b63ac6c47" |
|---|
| 211 | } |
|---|
| 212 | } |
|---|
| 213 | }, |
|---|
| 214 | "5381dda5-fd2a-4efd-8724-7c45c1f92c09": { |
|---|
| 215 | "id": "5381dda5-fd2a-4efd-8724-7c45c1f92c09", |
|---|
| 216 | "prev": "ec51efe5-ec7a-4b07-b785-fb71cabc2319", |
|---|
| 217 | "regions": { |
|---|
| 218 | "43c43c1a-872e-4494-85df-a7d10f9b0550": { |
|---|
| 219 | "attrs": { |
|---|
| 220 | "height": 0.8, |
|---|
| 221 | "width": 0.8, |
|---|
| 222 | "x": 0.1, |
|---|
| 223 | "y": 0.1 |
|---|
| 224 | }, |
|---|
| 225 | "content": { |
|---|
| 226 | "cell": "8a29bb08-cd2f-4c83-b858-15c44d56030e", |
|---|
| 227 | "part": "whole" |
|---|
| 228 | }, |
|---|
| 229 | "id": "43c43c1a-872e-4494-85df-a7d10f9b0550" |
|---|
| 230 | } |
|---|
| 231 | } |
|---|
| 232 | }, |
|---|
| 233 | "550158c0-8f2f-40c4-92cb-c10c8bf2f930": { |
|---|
| 234 | "id": "550158c0-8f2f-40c4-92cb-c10c8bf2f930", |
|---|
| 235 | "prev": "7427ba6e-da4a-4c0a-a357-799c13de2113", |
|---|
| 236 | "regions": { |
|---|
| 237 | "3e87230b-64d2-49cb-9527-3e26bff3fc6e": { |
|---|
| 238 | "attrs": { |
|---|
| 239 | "height": 0.8, |
|---|
| 240 | "width": 0.8, |
|---|
| 241 | "x": 0.1, |
|---|
| 242 | "y": 0.1 |
|---|
| 243 | }, |
|---|
| 244 | "content": { |
|---|
| 245 | "cell": "e5b6eb2f-46cb-4ad5-8732-7fd7792b42ba", |
|---|
| 246 | "part": "whole" |
|---|
| 247 | }, |
|---|
| 248 | "id": "3e87230b-64d2-49cb-9527-3e26bff3fc6e" |
|---|
| 249 | } |
|---|
| 250 | } |
|---|
| 251 | }, |
|---|
| 252 | "5b3a114a-ab45-43a4-8860-0245a98b26b9": { |
|---|
| 253 | "id": "5b3a114a-ab45-43a4-8860-0245a98b26b9", |
|---|
| 254 | "prev": "c7bf609c-1b48-4a11-8706-463abede978d", |
|---|
| 255 | "regions": { |
|---|
| 256 | "d70f470c-24d9-4c71-bd31-b0ff5a000dea": { |
|---|
| 257 | "attrs": { |
|---|
| 258 | "height": 0.8, |
|---|
| 259 | "width": 0.8, |
|---|
| 260 | "x": 0.1, |
|---|
| 261 | "y": 0.1 |
|---|
| 262 | }, |
|---|
| 263 | "content": { |
|---|
| 264 | "cell": "c7dc10b2-16ff-43df-9402-ce79b38f4d8c", |
|---|
| 265 | "part": "whole" |
|---|
| 266 | }, |
|---|
| 267 | "id": "d70f470c-24d9-4c71-bd31-b0ff5a000dea" |
|---|
| 268 | } |
|---|
| 269 | } |
|---|
| 270 | }, |
|---|
| 271 | "69429285-5c7f-4b7d-9243-ffc917b8f9ce": { |
|---|
| 272 | "id": "69429285-5c7f-4b7d-9243-ffc917b8f9ce", |
|---|
| 273 | "prev": "84264ee2-a515-4a95-b20c-491a0b8782d1", |
|---|
| 274 | "regions": { |
|---|
| 275 | "c9737b90-5ddc-42b1-a601-3c83808677eb": { |
|---|
| 276 | "attrs": { |
|---|
| 277 | "height": 0.8, |
|---|
| 278 | "width": 0.8, |
|---|
| 279 | "x": 0.1, |
|---|
| 280 | "y": 0.1 |
|---|
| 281 | }, |
|---|
| 282 | "content": { |
|---|
| 283 | "cell": "86272d04-2637-4206-885b-4873f4b50a80", |
|---|
| 284 | "part": "whole" |
|---|
| 285 | }, |
|---|
| 286 | "id": "c9737b90-5ddc-42b1-a601-3c83808677eb" |
|---|
| 287 | } |
|---|
| 288 | } |
|---|
| 289 | }, |
|---|
| 290 | "6a61d64d-530f-4f94-a29e-7bdf66bfaadb": { |
|---|
| 291 | "id": "6a61d64d-530f-4f94-a29e-7bdf66bfaadb", |
|---|
| 292 | "prev": "1ba27192-2b68-45b9-96f2-381167e1f808", |
|---|
| 293 | "regions": { |
|---|
| 294 | "6b39e02f-f47d-45eb-8c9a-74bdacfcefbd": { |
|---|
| 295 | "attrs": { |
|---|
| 296 | "height": 0.8, |
|---|
| 297 | "width": 0.8, |
|---|
| 298 | "x": 0.1, |
|---|
| 299 | "y": 0.1 |
|---|
| 300 | }, |
|---|
| 301 | "content": { |
|---|
| 302 | "cell": "7a8821c7-2bd8-4330-a54e-48f61b0516b7", |
|---|
| 303 | "part": "whole" |
|---|
| 304 | }, |
|---|
| 305 | "id": "6b39e02f-f47d-45eb-8c9a-74bdacfcefbd" |
|---|
| 306 | } |
|---|
| 307 | } |
|---|
| 308 | }, |
|---|
| 309 | "6e077d54-1b52-4b86-bcc3-43c44fd3f6e7": { |
|---|
| 310 | "id": "6e077d54-1b52-4b86-bcc3-43c44fd3f6e7", |
|---|
| 311 | "prev": "09c98d86-d894-4fd9-847e-1b89789b4e7f", |
|---|
| 312 | "regions": { |
|---|
| 313 | "93aee066-0ee3-491f-8ec5-7b73809d608f": { |
|---|
| 314 | "attrs": { |
|---|
| 315 | "height": 0.8, |
|---|
| 316 | "width": 0.8, |
|---|
| 317 | "x": 0.1, |
|---|
| 318 | "y": 0.1 |
|---|
| 319 | }, |
|---|
| 320 | "content": { |
|---|
| 321 | "cell": "96ff352e-308d-4ab7-a68e-244141128deb", |
|---|
| 322 | "part": "whole" |
|---|
| 323 | }, |
|---|
| 324 | "id": "93aee066-0ee3-491f-8ec5-7b73809d608f" |
|---|
| 325 | } |
|---|
| 326 | } |
|---|
| 327 | }, |
|---|
| 328 | "7427ba6e-da4a-4c0a-a357-799c13de2113": { |
|---|
| 329 | "id": "7427ba6e-da4a-4c0a-a357-799c13de2113", |
|---|
| 330 | "prev": "1b6bbadb-1200-45d3-a5e8-717091642c54", |
|---|
| 331 | "regions": { |
|---|
| 332 | "eb62c709-b1e2-4c5e-966b-c081b616e056": { |
|---|
| 333 | "attrs": { |
|---|
| 334 | "height": 0.8, |
|---|
| 335 | "width": 0.8, |
|---|
| 336 | "x": 0.1, |
|---|
| 337 | "y": 0.1 |
|---|
| 338 | }, |
|---|
| 339 | "content": { |
|---|
| 340 | "cell": "2ce12ebd-fed3-40ea-ade5-7c7031f112b0", |
|---|
| 341 | "part": "whole" |
|---|
| 342 | }, |
|---|
| 343 | "id": "eb62c709-b1e2-4c5e-966b-c081b616e056" |
|---|
| 344 | } |
|---|
| 345 | } |
|---|
| 346 | }, |
|---|
| 347 | "82ef1bf7-b0ce-4b1e-bc4c-c22909ee151b": { |
|---|
| 348 | "id": "82ef1bf7-b0ce-4b1e-bc4c-c22909ee151b", |
|---|
| 349 | "prev": "a303924c-421d-46f2-a6ad-d056562ce673", |
|---|
| 350 | "regions": { |
|---|
| 351 | "277ec992-07c7-4baa-a94c-1b76972aa2e7": { |
|---|
| 352 | "attrs": { |
|---|
| 353 | "height": 0.8, |
|---|
| 354 | "width": 0.8, |
|---|
| 355 | "x": 0.1, |
|---|
| 356 | "y": 0.1 |
|---|
| 357 | }, |
|---|
| 358 | "content": { |
|---|
| 359 | "cell": "2015e579-b211-40e4-a3cd-f0af5008b018", |
|---|
| 360 | "part": "whole" |
|---|
| 361 | }, |
|---|
| 362 | "id": "277ec992-07c7-4baa-a94c-1b76972aa2e7" |
|---|
| 363 | } |
|---|
| 364 | } |
|---|
| 365 | }, |
|---|
| 366 | "84264ee2-a515-4a95-b20c-491a0b8782d1": { |
|---|
| 367 | "id": "84264ee2-a515-4a95-b20c-491a0b8782d1", |
|---|
| 368 | "prev": "ddce1e58-49ae-43ce-8fec-2c4eefbe4150", |
|---|
| 369 | "regions": { |
|---|
| 370 | "3c987ec2-8841-4b25-92d2-1e9cdb8d22aa": { |
|---|
| 371 | "attrs": { |
|---|
| 372 | "height": 0.8, |
|---|
| 373 | "width": 0.8, |
|---|
| 374 | "x": 0.1, |
|---|
| 375 | "y": 0.1 |
|---|
| 376 | }, |
|---|
| 377 | "content": { |
|---|
| 378 | "cell": "122732e5-5b39-4ba8-9fbf-6834c9f6b085", |
|---|
| 379 | "part": "whole" |
|---|
| 380 | }, |
|---|
| 381 | "id": "3c987ec2-8841-4b25-92d2-1e9cdb8d22aa" |
|---|
| 382 | } |
|---|
| 383 | } |
|---|
| 384 | }, |
|---|
| 385 | "8f08e261-fb4a-4f9e-a021-23e6542c4a69": { |
|---|
| 386 | "id": "8f08e261-fb4a-4f9e-a021-23e6542c4a69", |
|---|
| 387 | "prev": "69429285-5c7f-4b7d-9243-ffc917b8f9ce", |
|---|
| 388 | "regions": { |
|---|
| 389 | "f80acf31-ec63-4ed8-a556-c02639b9febf": { |
|---|
| 390 | "attrs": { |
|---|
| 391 | "height": 0.8, |
|---|
| 392 | "width": 0.8, |
|---|
| 393 | "x": 0.1, |
|---|
| 394 | "y": 0.1 |
|---|
| 395 | }, |
|---|
| 396 | "content": { |
|---|
| 397 | "cell": "d233effc-9d8e-40df-9765-b53dfdff3c68", |
|---|
| 398 | "part": "whole" |
|---|
| 399 | }, |
|---|
| 400 | "id": "f80acf31-ec63-4ed8-a556-c02639b9febf" |
|---|
| 401 | } |
|---|
| 402 | } |
|---|
| 403 | }, |
|---|
| 404 | "9726092c-f997-4a46-bef8-d0ea1cc64a60": { |
|---|
| 405 | "id": "9726092c-f997-4a46-bef8-d0ea1cc64a60", |
|---|
| 406 | "prev": "f5da5529-f64c-4678-a224-aa88b1466107", |
|---|
| 407 | "regions": { |
|---|
| 408 | "6eba12aa-1328-44e4-b933-26cc140b4418": { |
|---|
| 409 | "attrs": { |
|---|
| 410 | "height": 0.8, |
|---|
| 411 | "width": 0.8, |
|---|
| 412 | "x": 0.1, |
|---|
| 413 | "y": 0.1 |
|---|
| 414 | }, |
|---|
| 415 | "content": { |
|---|
| 416 | "cell": "6e503bce-561b-49d9-883d-8b93c8f176ca", |
|---|
| 417 | "part": "whole" |
|---|
| 418 | }, |
|---|
| 419 | "id": "6eba12aa-1328-44e4-b933-26cc140b4418" |
|---|
| 420 | } |
|---|
| 421 | } |
|---|
| 422 | }, |
|---|
| 423 | "9c16251f-341a-47cd-8681-36b426db2f9b": { |
|---|
| 424 | "id": "9c16251f-341a-47cd-8681-36b426db2f9b", |
|---|
| 425 | "prev": "b3321028-c076-46dc-95cf-d0e796eb2e9a", |
|---|
| 426 | "regions": { |
|---|
| 427 | "294ce83c-a6fe-47f0-9e51-fac58920b355": { |
|---|
| 428 | "attrs": { |
|---|
| 429 | "height": 0.8, |
|---|
| 430 | "width": 0.8, |
|---|
| 431 | "x": 0.1, |
|---|
| 432 | "y": 0.1 |
|---|
| 433 | }, |
|---|
| 434 | "content": { |
|---|
| 435 | "cell": "c3764e84-53bf-40df-b60b-070fae9140ee", |
|---|
| 436 | "part": "whole" |
|---|
| 437 | }, |
|---|
| 438 | "id": "294ce83c-a6fe-47f0-9e51-fac58920b355" |
|---|
| 439 | } |
|---|
| 440 | } |
|---|
| 441 | }, |
|---|
| 442 | "9cf17728-2436-4f54-8c1c-4eafc4392021": { |
|---|
| 443 | "id": "9cf17728-2436-4f54-8c1c-4eafc4392021", |
|---|
| 444 | "prev": "550158c0-8f2f-40c4-92cb-c10c8bf2f930", |
|---|
| 445 | "regions": { |
|---|
| 446 | "d15d9370-cf49-4953-b12f-b925c0c2991d": { |
|---|
| 447 | "attrs": { |
|---|
| 448 | "height": 0.8, |
|---|
| 449 | "width": 0.8, |
|---|
| 450 | "x": 0.1, |
|---|
| 451 | "y": 0.1 |
|---|
| 452 | }, |
|---|
| 453 | "content": { |
|---|
| 454 | "cell": "b7a32c00-a4c7-49de-92fe-62ac6eae3a2e", |
|---|
| 455 | "part": "whole" |
|---|
| 456 | }, |
|---|
| 457 | "id": "d15d9370-cf49-4953-b12f-b925c0c2991d" |
|---|
| 458 | } |
|---|
| 459 | } |
|---|
| 460 | }, |
|---|
| 461 | "a303924c-421d-46f2-a6ad-d056562ce673": { |
|---|
| 462 | "id": "a303924c-421d-46f2-a6ad-d056562ce673", |
|---|
| 463 | "prev": "9cf17728-2436-4f54-8c1c-4eafc4392021", |
|---|
| 464 | "regions": { |
|---|
| 465 | "5863ab77-2ce5-4da3-b392-7b3628f45d6a": { |
|---|
| 466 | "attrs": { |
|---|
| 467 | "height": 0.8, |
|---|
| 468 | "width": 0.8, |
|---|
| 469 | "x": 0.1, |
|---|
| 470 | "y": 0.1 |
|---|
| 471 | }, |
|---|
| 472 | "content": { |
|---|
| 473 | "cell": "2a521a06-187b-43fc-b7cb-8ce3f56e0a13", |
|---|
| 474 | "part": "whole" |
|---|
| 475 | }, |
|---|
| 476 | "id": "5863ab77-2ce5-4da3-b392-7b3628f45d6a" |
|---|
| 477 | } |
|---|
| 478 | } |
|---|
| 479 | }, |
|---|
| 480 | "af9260f5-c3c0-4457-8946-fc08623fdff8": { |
|---|
| 481 | "id": "af9260f5-c3c0-4457-8946-fc08623fdff8", |
|---|
| 482 | "prev": "24da9cf4-1ecd-4314-a35e-fa3ed8056970", |
|---|
| 483 | "regions": { |
|---|
| 484 | "0607cc59-9de9-4fec-9a0b-cc94c72b6726": { |
|---|
| 485 | "attrs": { |
|---|
| 486 | "height": 0.8, |
|---|
| 487 | "width": 0.8, |
|---|
| 488 | "x": 0.1, |
|---|
| 489 | "y": 0.1 |
|---|
| 490 | }, |
|---|
| 491 | "content": { |
|---|
| 492 | "cell": "de5b327b-9089-482f-ae6f-39e6fe3dac32", |
|---|
| 493 | "part": "whole" |
|---|
| 494 | }, |
|---|
| 495 | "id": "0607cc59-9de9-4fec-9a0b-cc94c72b6726" |
|---|
| 496 | } |
|---|
| 497 | } |
|---|
| 498 | }, |
|---|
| 499 | "b3321028-c076-46dc-95cf-d0e796eb2e9a": { |
|---|
| 500 | "id": "b3321028-c076-46dc-95cf-d0e796eb2e9a", |
|---|
| 501 | "prev": "2c78a52b-56af-4232-ab1b-16abbe0a79bc", |
|---|
| 502 | "regions": { |
|---|
| 503 | "9914a8c1-c8a1-431c-b3c3-4eb766c2d453": { |
|---|
| 504 | "attrs": { |
|---|
| 505 | "height": 0.8, |
|---|
| 506 | "width": 0.8, |
|---|
| 507 | "x": 0.1, |
|---|
| 508 | "y": 0.1 |
|---|
| 509 | }, |
|---|
| 510 | "content": { |
|---|
| 511 | "cell": "64ab734f-91f4-45c1-82a7-ce2cfbf80b2f", |
|---|
| 512 | "part": "whole" |
|---|
| 513 | }, |
|---|
| 514 | "id": "9914a8c1-c8a1-431c-b3c3-4eb766c2d453" |
|---|
| 515 | } |
|---|
| 516 | } |
|---|
| 517 | }, |
|---|
| 518 | "ba9d22c2-c855-4506-9d76-3609d7331ae2": { |
|---|
| 519 | "id": "ba9d22c2-c855-4506-9d76-3609d7331ae2", |
|---|
| 520 | "prev": "82ef1bf7-b0ce-4b1e-bc4c-c22909ee151b", |
|---|
| 521 | "regions": { |
|---|
| 522 | "956bdaae-0aa8-4cfa-b9c8-6f208b2e2711": { |
|---|
| 523 | "attrs": { |
|---|
| 524 | "height": 0.8, |
|---|
| 525 | "width": 0.8, |
|---|
| 526 | "x": 0.1, |
|---|
| 527 | "y": 0.1 |
|---|
| 528 | }, |
|---|
| 529 | "content": { |
|---|
| 530 | "cell": "6cd5d476-5808-4544-a652-d70c566e51c6", |
|---|
| 531 | "part": "whole" |
|---|
| 532 | }, |
|---|
| 533 | "id": "956bdaae-0aa8-4cfa-b9c8-6f208b2e2711" |
|---|
| 534 | } |
|---|
| 535 | } |
|---|
| 536 | }, |
|---|
| 537 | "c1674358-a4a8-49d6-8c9e-b9b654dec8d1": { |
|---|
| 538 | "id": "c1674358-a4a8-49d6-8c9e-b9b654dec8d1", |
|---|
| 539 | "prev": "ba9d22c2-c855-4506-9d76-3609d7331ae2", |
|---|
| 540 | "regions": { |
|---|
| 541 | "3ce06210-7b25-4f08-a488-aa45acb1eb25": { |
|---|
| 542 | "attrs": { |
|---|
| 543 | "height": 0.8, |
|---|
| 544 | "width": 0.8, |
|---|
| 545 | "x": 0.1, |
|---|
| 546 | "y": 0.1 |
|---|
| 547 | }, |
|---|
| 548 | "content": { |
|---|
| 549 | "cell": "f34815e0-c51e-42d5-a17d-04a497c6e17b", |
|---|
| 550 | "part": "whole" |
|---|
| 551 | }, |
|---|
| 552 | "id": "3ce06210-7b25-4f08-a488-aa45acb1eb25" |
|---|
| 553 | } |
|---|
| 554 | } |
|---|
| 555 | }, |
|---|
| 556 | "c24ac2b3-9975-4ee4-b59c-df7f94db8722": { |
|---|
| 557 | "id": "c24ac2b3-9975-4ee4-b59c-df7f94db8722", |
|---|
| 558 | "prev": "5381dda5-fd2a-4efd-8724-7c45c1f92c09", |
|---|
| 559 | "regions": { |
|---|
| 560 | "fa876f10-c1e4-4cd5-b136-bf7e86ec8e93": { |
|---|
| 561 | "attrs": { |
|---|
| 562 | "height": 0.8, |
|---|
| 563 | "width": 0.8, |
|---|
| 564 | "x": 0.1, |
|---|
| 565 | "y": 0.1 |
|---|
| 566 | }, |
|---|
| 567 | "content": { |
|---|
| 568 | "cell": "c0a1ebc8-217f-4d49-9bcd-eeca4debf726", |
|---|
| 569 | "part": "whole" |
|---|
| 570 | }, |
|---|
| 571 | "id": "fa876f10-c1e4-4cd5-b136-bf7e86ec8e93" |
|---|
| 572 | } |
|---|
| 573 | } |
|---|
| 574 | }, |
|---|
| 575 | "c7bf609c-1b48-4a11-8706-463abede978d": { |
|---|
| 576 | "id": "c7bf609c-1b48-4a11-8706-463abede978d", |
|---|
| 577 | "prev": "9c16251f-341a-47cd-8681-36b426db2f9b", |
|---|
| 578 | "regions": { |
|---|
| 579 | "af05ade8-5cca-41a1-b602-4aa86f78c2af": { |
|---|
| 580 | "attrs": { |
|---|
| 581 | "height": 0.8, |
|---|
| 582 | "width": 0.8, |
|---|
| 583 | "x": 0.1, |
|---|
| 584 | "y": 0.1 |
|---|
| 585 | }, |
|---|
| 586 | "content": { |
|---|
| 587 | "cell": "69c017ce-9d2f-4e23-9ceb-36e8a09889e0", |
|---|
| 588 | "part": "whole" |
|---|
| 589 | }, |
|---|
| 590 | "id": "af05ade8-5cca-41a1-b602-4aa86f78c2af" |
|---|
| 591 | } |
|---|
| 592 | } |
|---|
| 593 | }, |
|---|
| 594 | "dab15b59-64b4-4c70-b208-dfdc24a948aa": { |
|---|
| 595 | "id": "dab15b59-64b4-4c70-b208-dfdc24a948aa", |
|---|
| 596 | "prev": "6e077d54-1b52-4b86-bcc3-43c44fd3f6e7", |
|---|
| 597 | "regions": { |
|---|
| 598 | "712cca5c-f412-4c2d-8507-66ea68041569": { |
|---|
| 599 | "attrs": { |
|---|
| 600 | "height": 0.8, |
|---|
| 601 | "width": 0.8, |
|---|
| 602 | "x": 0.1, |
|---|
| 603 | "y": 0.1 |
|---|
| 604 | }, |
|---|
| 605 | "content": { |
|---|
| 606 | "cell": "020e99ab-4d42-47c9-b637-d2b7dc72285c", |
|---|
| 607 | "part": "whole" |
|---|
| 608 | }, |
|---|
| 609 | "id": "712cca5c-f412-4c2d-8507-66ea68041569" |
|---|
| 610 | } |
|---|
| 611 | } |
|---|
| 612 | }, |
|---|
| 613 | "ddce1e58-49ae-43ce-8fec-2c4eefbe4150": { |
|---|
| 614 | "id": "ddce1e58-49ae-43ce-8fec-2c4eefbe4150", |
|---|
| 615 | "prev": "e2bc7986-85cd-4e1d-b58e-b88847240960", |
|---|
| 616 | "regions": { |
|---|
| 617 | "e3925122-fe2c-46fe-bf26-5685cf07236d": { |
|---|
| 618 | "attrs": { |
|---|
| 619 | "height": 0.8, |
|---|
| 620 | "width": 0.8, |
|---|
| 621 | "x": 0.1, |
|---|
| 622 | "y": 0.1 |
|---|
| 623 | }, |
|---|
| 624 | "content": { |
|---|
| 625 | "cell": "c5c51550-7869-4d63-a72e-73a3305c82f7", |
|---|
| 626 | "part": "whole" |
|---|
| 627 | }, |
|---|
| 628 | "id": "e3925122-fe2c-46fe-bf26-5685cf07236d" |
|---|
| 629 | } |
|---|
| 630 | } |
|---|
| 631 | }, |
|---|
| 632 | "e2bc7986-85cd-4e1d-b58e-b88847240960": { |
|---|
| 633 | "id": "e2bc7986-85cd-4e1d-b58e-b88847240960", |
|---|
| 634 | "prev": "fce974f8-f2a7-4753-8aea-fcc4fbb83df1", |
|---|
| 635 | "regions": { |
|---|
| 636 | "e59ee17e-ed54-45f5-b48a-2c251c4685fe": { |
|---|
| 637 | "attrs": { |
|---|
| 638 | "height": 0.8, |
|---|
| 639 | "width": 0.8, |
|---|
| 640 | "x": 0.1, |
|---|
| 641 | "y": 0.1 |
|---|
| 642 | }, |
|---|
| 643 | "content": { |
|---|
| 644 | "cell": "8950d61e-500f-4004-917a-6613dec78eac", |
|---|
| 645 | "part": "whole" |
|---|
| 646 | }, |
|---|
| 647 | "id": "e59ee17e-ed54-45f5-b48a-2c251c4685fe" |
|---|
| 648 | } |
|---|
| 649 | } |
|---|
| 650 | }, |
|---|
| 651 | "e81dcf87-fb95-4d33-89f9-6d820212ff9e": { |
|---|
| 652 | "id": "e81dcf87-fb95-4d33-89f9-6d820212ff9e", |
|---|
| 653 | "prev": "190227c4-a075-4074-ae84-e9ddf8e69560", |
|---|
| 654 | "regions": { |
|---|
| 655 | "f7ec68ba-436f-42e5-aa80-3c24042a5632": { |
|---|
| 656 | "attrs": { |
|---|
| 657 | "height": 0.8, |
|---|
| 658 | "width": 0.8, |
|---|
| 659 | "x": 0.1, |
|---|
| 660 | "y": 0.1 |
|---|
| 661 | }, |
|---|
| 662 | "content": { |
|---|
| 663 | "cell": "f5a8fe74-4b1e-4ad3-b45b-344ae29f540e", |
|---|
| 664 | "part": "whole" |
|---|
| 665 | }, |
|---|
| 666 | "id": "f7ec68ba-436f-42e5-aa80-3c24042a5632" |
|---|
| 667 | } |
|---|
| 668 | } |
|---|
| 669 | }, |
|---|
| 670 | "eb9a3413-8bc3-4e0b-b188-ef05e5f0c461": { |
|---|
| 671 | "id": "eb9a3413-8bc3-4e0b-b188-ef05e5f0c461", |
|---|
| 672 | "prev": "c1674358-a4a8-49d6-8c9e-b9b654dec8d1", |
|---|
| 673 | "regions": { |
|---|
| 674 | "e0ec3cdf-6b8b-44dc-88e4-ae4ad5bb151c": { |
|---|
| 675 | "attrs": { |
|---|
| 676 | "height": 0.8, |
|---|
| 677 | "width": 0.8, |
|---|
| 678 | "x": 0.1, |
|---|
| 679 | "y": 0.1 |
|---|
| 680 | }, |
|---|
| 681 | "content": { |
|---|
| 682 | "cell": "51434e41-a231-41e1-a087-1e788710ef58", |
|---|
| 683 | "part": "whole" |
|---|
| 684 | }, |
|---|
| 685 | "id": "e0ec3cdf-6b8b-44dc-88e4-ae4ad5bb151c" |
|---|
| 686 | } |
|---|
| 687 | } |
|---|
| 688 | }, |
|---|
| 689 | "ec51efe5-ec7a-4b07-b785-fb71cabc2319": { |
|---|
| 690 | "id": "ec51efe5-ec7a-4b07-b785-fb71cabc2319", |
|---|
| 691 | "prev": "004b13a1-f767-463b-80f7-8599c4e09e5a", |
|---|
| 692 | "regions": { |
|---|
| 693 | "486a07de-4f1a-4d49-b42b-5164ae74c3ae": { |
|---|
| 694 | "attrs": { |
|---|
| 695 | "height": 0.8, |
|---|
| 696 | "width": 0.8, |
|---|
| 697 | "x": 0.1, |
|---|
| 698 | "y": 0.1 |
|---|
| 699 | }, |
|---|
| 700 | "content": { |
|---|
| 701 | "cell": "cf2caeae-95b8-45d3-8f78-6293cf823920", |
|---|
| 702 | "part": "whole" |
|---|
| 703 | }, |
|---|
| 704 | "id": "486a07de-4f1a-4d49-b42b-5164ae74c3ae" |
|---|
| 705 | } |
|---|
| 706 | } |
|---|
| 707 | }, |
|---|
| 708 | "f066aa28-05ba-4dde-81e2-7a9b7f104c55": { |
|---|
| 709 | "id": "f066aa28-05ba-4dde-81e2-7a9b7f104c55", |
|---|
| 710 | "prev": "5b3a114a-ab45-43a4-8860-0245a98b26b9", |
|---|
| 711 | "regions": { |
|---|
| 712 | "7f0b334e-0138-480d-9f9f-5394ca004335": { |
|---|
| 713 | "attrs": { |
|---|
| 714 | "height": 0.8, |
|---|
| 715 | "width": 0.8, |
|---|
| 716 | "x": 0.1, |
|---|
| 717 | "y": 0.1 |
|---|
| 718 | }, |
|---|
| 719 | "content": { |
|---|
| 720 | "cell": "d63764e7-e961-4508-9148-263112f0f5a4", |
|---|
| 721 | "part": "whole" |
|---|
| 722 | }, |
|---|
| 723 | "id": "7f0b334e-0138-480d-9f9f-5394ca004335" |
|---|
| 724 | } |
|---|
| 725 | } |
|---|
| 726 | }, |
|---|
| 727 | "f5da5529-f64c-4678-a224-aa88b1466107": { |
|---|
| 728 | "id": "f5da5529-f64c-4678-a224-aa88b1466107", |
|---|
| 729 | "prev": "375c8fec-6f3b-4b01-89ad-410c4f27a4d0", |
|---|
| 730 | "regions": { |
|---|
| 731 | "84471409-37dd-4cbf-a258-ef06e4e8d072": { |
|---|
| 732 | "attrs": { |
|---|
| 733 | "height": 0.8, |
|---|
| 734 | "width": 0.8, |
|---|
| 735 | "x": 0.1, |
|---|
| 736 | "y": 0.1 |
|---|
| 737 | }, |
|---|
| 738 | "content": { |
|---|
| 739 | "cell": "9fcfcdbd-7424-4cdc-a7be-23d241c7a20a", |
|---|
| 740 | "part": "whole" |
|---|
| 741 | }, |
|---|
| 742 | "id": "84471409-37dd-4cbf-a258-ef06e4e8d072" |
|---|
| 743 | } |
|---|
| 744 | } |
|---|
| 745 | }, |
|---|
| 746 | "fce974f8-f2a7-4753-8aea-fcc4fbb83df1": { |
|---|
| 747 | "id": "fce974f8-f2a7-4753-8aea-fcc4fbb83df1", |
|---|
| 748 | "prev": "3feede8b-0bbd-42b9-915c-56e40727cf62", |
|---|
| 749 | "regions": { |
|---|
| 750 | "26c6a0b6-d747-4851-a539-42ffd65b2be7": { |
|---|
| 751 | "attrs": { |
|---|
| 752 | "height": 0.8, |
|---|
| 753 | "width": 0.8, |
|---|
| 754 | "x": 0.1, |
|---|
| 755 | "y": 0.1 |
|---|
| 756 | }, |
|---|
| 757 | "content": { |
|---|
| 758 | "cell": "ee81f282-5b75-4077-bbf5-c4632d0a2d96", |
|---|
| 759 | "part": "whole" |
|---|
| 760 | }, |
|---|
| 761 | "id": "26c6a0b6-d747-4851-a539-42ffd65b2be7" |
|---|
| 762 | } |
|---|
| 763 | } |
|---|
| 764 | } |
|---|
| 765 | }, |
|---|
| 766 | "themes": {} |
|---|
| 767 | }, |
|---|
| 768 | "colab": { |
|---|
| 769 | "name": "Word_Vectors_and_Sentiment.ipynb", |
|---|
| 770 | "provenance": [], |
|---|
| 771 | "collapsed_sections": [] |
|---|
| 772 | } |
|---|
| 773 | }, |
|---|
| 774 | "cells": [ |
|---|
| 775 | { |
|---|
| 776 | "cell_type": "markdown", |
|---|
| 777 | "metadata": { |
|---|
| 778 | "nbpresent": { |
|---|
| 779 | "id": "f3dbef4d-9bf4-4e24-bcd0-a4ce3dd00fb8" |
|---|
| 780 | }, |
|---|
| 781 | "id": "bZslM_O1FZx6" |
|---|
| 782 | }, |
|---|
| 783 | "source": [ |
|---|
| 784 | "# Sentiment Analysis Notebook\n", |
|---|
| 785 | "\n", |
|---|
| 786 | "Inspired by https://www.kaggle.com/gabrielaltay/word-vectors-from-pmi-matrix and https://www.kaggle.com/rosado/sentiment-analysis-text-mining.\n", |
|---|
| 787 | "\n", |
|---|
| 788 | "The notebook can handle two different datasets: Cestina 2.0 and Urban Dictionary. Select the one close to your language knowledge. Later, you can try with the other one.\n", |
|---|
| 789 | "\n", |
|---|
| 790 | "The aim of this notebook is to demonstrate sentiment analysis, particularly on new words from the crowd-sourced website http://cestina20.cz or https://www.urbandictionary.com/. We scraped the data from Cestina 2.0 into a CSV file that is part of this project - http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/cestina20.csv. In case of Urban Dictionary, we downloaded the CSV from Kaggle: https://www.kaggle.com/athontz/urban-dictionary-terms#urban_dictionary.csv\n", |
|---|
| 791 | "\n", |
|---|
| 792 | "First, we try to recognize the sentiment of dictionary entries using the Liu's Opinion Lexicon (https://www.cs.uic.edu/~liub/FBS/sentiment-analysis.html#lexicon). For Czech, we automatically translated the entries using Google Translate. The Opinion Lexicon as well as the translation are part of this project and can be found in http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/positive-words-en.txt, http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/negative-words-en.txt, http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/positive-words-cs.txt, and http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/negative-words-cs.txt.\n", |
|---|
| 793 | "\n", |
|---|
| 794 | "**OPTIONAL TASK**: If you are not familiar with Cestina 2.0/Urban Dictionary, go to the website and go through some dictionary entries to see example of the data.\n" |
|---|
| 795 | ] |
|---|
| 796 | }, |
|---|
| 797 | { |
|---|
| 798 | "cell_type": "markdown", |
|---|
| 799 | "metadata": { |
|---|
| 800 | "id": "liOlaHucHpYX" |
|---|
| 801 | }, |
|---|
| 802 | "source": [ |
|---|
| 803 | "# Get the data\n", |
|---|
| 804 | "\n", |
|---|
| 805 | "Download the data using `wget`." |
|---|
| 806 | ] |
|---|
| 807 | }, |
|---|
| 808 | { |
|---|
| 809 | "cell_type": "code", |
|---|
| 810 | "metadata": { |
|---|
| 811 | "id": "GqX7odWuH0-O" |
|---|
| 812 | }, |
|---|
| 813 | "source": [ |
|---|
| 814 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/cestina20.csv\n", |
|---|
| 815 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/urban_dictionary.csv\n", |
|---|
| 816 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/positive-words-en.txt\n", |
|---|
| 817 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/negative-words-en.txt\n", |
|---|
| 818 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/positive-words-cs.txt\n", |
|---|
| 819 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/negative-words-cs.txt" |
|---|
| 820 | ], |
|---|
| 821 | "execution_count": null, |
|---|
| 822 | "outputs": [] |
|---|
| 823 | }, |
|---|
| 824 | { |
|---|
| 825 | "cell_type": "markdown", |
|---|
| 826 | "metadata": { |
|---|
| 827 | "nbpresent": { |
|---|
| 828 | "id": "96ff352e-308d-4ab7-a68e-244141128deb" |
|---|
| 829 | }, |
|---|
| 830 | "id": "DLi34YEjFZx7" |
|---|
| 831 | }, |
|---|
| 832 | "source": [ |
|---|
| 833 | "## Tools\n", |
|---|
| 834 | "### Tokenizer\n", |
|---|
| 835 | "We use NLTK standard tokenizer to split the texts by words. Splitting by spaces is not enough, since we want e.g. \"word\" and \"word,\" to be one token. Tokenization is not strongly language dependent, so NLTK standard tokenizer is enough. If we want to process languages that do not use spaces (CJK, or Chinese, Japanese, Korean), we should modify this part.\n", |
|---|
| 836 | "\n", |
|---|
| 837 | "### Stopwords\n", |
|---|
| 838 | "For training the word vectors, we use stoplists of English/Czech most common words. This helps especially in cases we have small data (our case)." |
|---|
| 839 | ] |
|---|
| 840 | }, |
|---|
| 841 | { |
|---|
| 842 | "cell_type": "code", |
|---|
| 843 | "metadata": { |
|---|
| 844 | "nbpresent": { |
|---|
| 845 | "id": "020e99ab-4d42-47c9-b637-d2b7dc72285c" |
|---|
| 846 | }, |
|---|
| 847 | "id": "g4-0vj0fFZx8" |
|---|
| 848 | }, |
|---|
| 849 | "source": [ |
|---|
| 850 | "!wget https://raw.githubusercontent.com/stopwords-iso/stopwords-cs/master/stopwords-cs.txt\n", |
|---|
| 851 | "!wget https://raw.githubusercontent.com/stopwords-iso/stopwords-en/master/stopwords-en.txt" |
|---|
| 852 | ], |
|---|
| 853 | "execution_count": null, |
|---|
| 854 | "outputs": [] |
|---|
| 855 | }, |
|---|
| 856 | { |
|---|
| 857 | "cell_type": "markdown", |
|---|
| 858 | "metadata": { |
|---|
| 859 | "nbpresent": { |
|---|
| 860 | "id": "60ff129a-80bd-4c96-8f5e-8c2284f71f1c" |
|---|
| 861 | }, |
|---|
| 862 | "id": "TB8JNSlQFZx_" |
|---|
| 863 | }, |
|---|
| 864 | "source": [ |
|---|
| 865 | "### Python packages\n", |
|---|
| 866 | "#### Pandas\n", |
|---|
| 867 | "Pandas is a data science standard that allows easy work with large tabular data. Pandas DataFrame is the object we use in this project.\n", |
|---|
| 868 | "\n", |
|---|
| 869 | "#### SciKit Learn\n", |
|---|
| 870 | "SciKit Learn (`sklearn`) is a standard machine learning package for Python. We use its `cosine_similarity` function.\n", |
|---|
| 871 | "\n", |
|---|
| 872 | "#### Numpy\n", |
|---|
| 873 | "Together with `sklearn` a Python machine learning standard. Provides straightforward matrix computation, so we avoid unhealthy nested for-cycles." |
|---|
| 874 | ] |
|---|
| 875 | }, |
|---|
| 876 | { |
|---|
| 877 | "cell_type": "code", |
|---|
| 878 | "metadata": { |
|---|
| 879 | "id": "9xx0fsI7FZyA" |
|---|
| 880 | }, |
|---|
| 881 | "source": [ |
|---|
| 882 | "# it depends on installation but probably this is not necessary\n", |
|---|
| 883 | "# in case it does not work, try pip instead of pip3\n", |
|---|
| 884 | "# DO NOT RUN in Colab, this is only useful if you download the notebook and use on your computer\n", |
|---|
| 885 | "!pip3 install --user nltk\n", |
|---|
| 886 | "!pip3 install --user sklearn" |
|---|
| 887 | ], |
|---|
| 888 | "execution_count": null, |
|---|
| 889 | "outputs": [] |
|---|
| 890 | }, |
|---|
| 891 | { |
|---|
| 892 | "cell_type": "code", |
|---|
| 893 | "metadata": { |
|---|
| 894 | "nbpresent": { |
|---|
| 895 | "id": "2ce12ebd-fed3-40ea-ade5-7c7031f112b0" |
|---|
| 896 | }, |
|---|
| 897 | "id": "MGeYB8WUFZyC" |
|---|
| 898 | }, |
|---|
| 899 | "source": [ |
|---|
| 900 | "from collections import Counter\n", |
|---|
| 901 | "import itertools\n", |
|---|
| 902 | "\n", |
|---|
| 903 | "import os\n", |
|---|
| 904 | "import nltk\n", |
|---|
| 905 | "nltk.download('punkt')\n", |
|---|
| 906 | "from nltk.tokenize import word_tokenize\n", |
|---|
| 907 | "\n", |
|---|
| 908 | "import numpy as np\n", |
|---|
| 909 | "import pandas as pd\n", |
|---|
| 910 | "from scipy import sparse\n", |
|---|
| 911 | "from sklearn.preprocessing import normalize\n", |
|---|
| 912 | "from sklearn.metrics.pairwise import cosine_similarity" |
|---|
| 913 | ], |
|---|
| 914 | "execution_count": null, |
|---|
| 915 | "outputs": [] |
|---|
| 916 | }, |
|---|
| 917 | { |
|---|
| 918 | "cell_type": "code", |
|---|
| 919 | "metadata": { |
|---|
| 920 | "nbpresent": { |
|---|
| 921 | "id": "e5b6eb2f-46cb-4ad5-8732-7fd7792b42ba" |
|---|
| 922 | }, |
|---|
| 923 | "id": "5q0WbMHTFZyF" |
|---|
| 924 | }, |
|---|
| 925 | "source": [ |
|---|
| 926 | "LANG_SETTINGS = {\"en\": {\"filename\":\"urban_dictionary.csv\", \"sep\":\",\", \"explanation\":\"definition\"},\n", |
|---|
| 927 | " \"cs\": {\"filename\":\"cestina20.csv\", \"sep\":\";\", \"explanation\":\"explanation\"}\n", |
|---|
| 928 | " }\n", |
|---|
| 929 | "selected_language = 'cs'\n", |
|---|
| 930 | "settings = LANG_SETTINGS[selected_language]" |
|---|
| 931 | ], |
|---|
| 932 | "execution_count": null, |
|---|
| 933 | "outputs": [] |
|---|
| 934 | }, |
|---|
| 935 | { |
|---|
| 936 | "cell_type": "markdown", |
|---|
| 937 | "metadata": { |
|---|
| 938 | "nbpresent": { |
|---|
| 939 | "id": "b7a32c00-a4c7-49de-92fe-62ac6eae3a2e" |
|---|
| 940 | }, |
|---|
| 941 | "id": "nvI2bSxjFZyI" |
|---|
| 942 | }, |
|---|
| 943 | "source": [ |
|---|
| 944 | "## Data\n", |
|---|
| 945 | "We prepared the data in advance using web crawling and parsing of the HTML pages. Look at a sample of the CSV." |
|---|
| 946 | ] |
|---|
| 947 | }, |
|---|
| 948 | { |
|---|
| 949 | "cell_type": "code", |
|---|
| 950 | "metadata": { |
|---|
| 951 | "nbpresent": { |
|---|
| 952 | "id": "2a521a06-187b-43fc-b7cb-8ce3f56e0a13" |
|---|
| 953 | }, |
|---|
| 954 | "id": "QQYDZrYqFZyI" |
|---|
| 955 | }, |
|---|
| 956 | "source": [ |
|---|
| 957 | "df = pd.read_csv(settings['filename'], sep=settings['sep'], keep_default_na=False)\n", |
|---|
| 958 | "df.head()" |
|---|
| 959 | ], |
|---|
| 960 | "execution_count": null, |
|---|
| 961 | "outputs": [] |
|---|
| 962 | }, |
|---|
| 963 | { |
|---|
| 964 | "cell_type": "markdown", |
|---|
| 965 | "metadata": { |
|---|
| 966 | "nbpresent": { |
|---|
| 967 | "id": "2015e579-b211-40e4-a3cd-f0af5008b018" |
|---|
| 968 | }, |
|---|
| 969 | "id": "t8jNI6zHFZyL" |
|---|
| 970 | }, |
|---|
| 971 | "source": [ |
|---|
| 972 | "### Explanations\n", |
|---|
| 973 | "We try to recognize dictionary entry sentiment from the explanation. Let's see an example, next, let's convert the explanations into token sequences and remove stopwords." |
|---|
| 974 | ] |
|---|
| 975 | }, |
|---|
| 976 | { |
|---|
| 977 | "cell_type": "code", |
|---|
| 978 | "metadata": { |
|---|
| 979 | "nbpresent": { |
|---|
| 980 | "id": "6cd5d476-5808-4544-a652-d70c566e51c6" |
|---|
| 981 | }, |
|---|
| 982 | "id": "WXEMVowlFZyM" |
|---|
| 983 | }, |
|---|
| 984 | "source": [ |
|---|
| 985 | "df[settings['explanation']][1000]" |
|---|
| 986 | ], |
|---|
| 987 | "execution_count": null, |
|---|
| 988 | "outputs": [] |
|---|
| 989 | }, |
|---|
| 990 | { |
|---|
| 991 | "cell_type": "code", |
|---|
| 992 | "metadata": { |
|---|
| 993 | "nbpresent": { |
|---|
| 994 | "id": "f34815e0-c51e-42d5-a17d-04a497c6e17b" |
|---|
| 995 | }, |
|---|
| 996 | "id": "OGkFY7XfFZyP" |
|---|
| 997 | }, |
|---|
| 998 | "source": [ |
|---|
| 999 | "explanations = df[settings['explanation']].tolist()\n", |
|---|
| 1000 | "# remove stopwords\n", |
|---|
| 1001 | "stopwords_set = ['-','.',':',';','\"',',', '!', '(', ')', '``', \"'\", \"''\", \"„\", \"”\", \"...\", \"apod\", \"viz\", 'např', \"například\", \"příklad\"]\n", |
|---|
| 1002 | "with open('stopwords-{}.txt'.format(selected_language), encoding='utf-8') as f:\n", |
|---|
| 1003 | " stopwords_set.extend(list(set([w.strip() for w in f.readlines()])))\n", |
|---|
| 1004 | "#print(stopwords_set)\n", |
|---|
| 1005 | "explanations = [\n", |
|---|
| 1006 | " [tok.lower() for tok in word_tokenize(explanation.replace('…', '...').replace('…', '...')) if tok.lower() not in stopwords_set] for explanation in explanations\n", |
|---|
| 1007 | "]\n", |
|---|
| 1008 | "# show results\n", |
|---|
| 1009 | "explanations[1000]" |
|---|
| 1010 | ], |
|---|
| 1011 | "execution_count": null, |
|---|
| 1012 | "outputs": [] |
|---|
| 1013 | }, |
|---|
| 1014 | { |
|---|
| 1015 | "cell_type": "markdown", |
|---|
| 1016 | "metadata": { |
|---|
| 1017 | "nbpresent": { |
|---|
| 1018 | "id": "51434e41-a231-41e1-a087-1e788710ef58" |
|---|
| 1019 | }, |
|---|
| 1020 | "id": "5LVQdGIhFZyS" |
|---|
| 1021 | }, |
|---|
| 1022 | "source": [ |
|---|
| 1023 | "## Recognize sentiment using the Opinion Lexicon\n", |
|---|
| 1024 | "\n", |
|---|
| 1025 | "**TASK 1**: Look in the original opinion lexicon, look in the translated version. Comment on what you see in both resources.\n", |
|---|
| 1026 | "\n", |
|---|
| 1027 | "The easiest way is to go through all tokens in the explanation and sum their sentiment according to the Opinion Lexicon. Since some opinion lexicons distinguish *strong* and *weak* opinions we convert the input data into positive or negative numbers in a similar way, except we do not distinguish the intensity of the sentiment, only the polarity: -2 for negative words, 2 for positive words, 0 for words present in both files.\n" |
|---|
| 1028 | ] |
|---|
| 1029 | }, |
|---|
| 1030 | { |
|---|
| 1031 | "cell_type": "code", |
|---|
| 1032 | "metadata": { |
|---|
| 1033 | "nbpresent": { |
|---|
| 1034 | "id": "74d938a4-4600-41ae-8cab-dd359e64cfa5" |
|---|
| 1035 | }, |
|---|
| 1036 | "scrolled": true, |
|---|
| 1037 | "id": "48ljxlL5FZyS" |
|---|
| 1038 | }, |
|---|
| 1039 | "source": [ |
|---|
| 1040 | "with open('positive-words-{}.txt'.format(selected_language),'rb') as f:\n", |
|---|
| 1041 | " positive_words = [w.strip().lower() for w in f.read().decode('utf-8','ignore').split('\\n')]\n", |
|---|
| 1042 | "with open('negative-words-{}.txt'.format(selected_language),'rb') as f:\n", |
|---|
| 1043 | " negative_words = [w.strip().lower() for w in f.read().decode('utf-8','ignore').split('\\n')]\n", |
|---|
| 1044 | "\"No. of positive words:\", len(positive_words), \"No. of negative words:\", len(negative_words)" |
|---|
| 1045 | ], |
|---|
| 1046 | "execution_count": null, |
|---|
| 1047 | "outputs": [] |
|---|
| 1048 | }, |
|---|
| 1049 | { |
|---|
| 1050 | "cell_type": "code", |
|---|
| 1051 | "metadata": { |
|---|
| 1052 | "nbpresent": { |
|---|
| 1053 | "id": "64ab734f-91f4-45c1-82a7-ce2cfbf80b2f" |
|---|
| 1054 | }, |
|---|
| 1055 | "id": "iVW0Dy_FFZyU" |
|---|
| 1056 | }, |
|---|
| 1057 | "source": [ |
|---|
| 1058 | " # we put a small epsilon in order to distinguish sentences with no recognized sentiment from sentences with positive+negative sentiment\n", |
|---|
| 1059 | "score_word_dict = {k:2.00001 for k in positive_words}\n", |
|---|
| 1060 | "score_word_dict.update({k:0 for k in negative_words if k in positive_words})\n", |
|---|
| 1061 | "score_word_dict.update({k:-2 for k in negative_words if k not in positive_words})" |
|---|
| 1062 | ], |
|---|
| 1063 | "execution_count": null, |
|---|
| 1064 | "outputs": [] |
|---|
| 1065 | }, |
|---|
| 1066 | { |
|---|
| 1067 | "cell_type": "code", |
|---|
| 1068 | "metadata": { |
|---|
| 1069 | "nbpresent": { |
|---|
| 1070 | "id": "c3764e84-53bf-40df-b60b-070fae9140ee" |
|---|
| 1071 | }, |
|---|
| 1072 | "id": "iK7_-EeIFZyW" |
|---|
| 1073 | }, |
|---|
| 1074 | "source": [ |
|---|
| 1075 | "def get_sentiment_sequence(tokens, score_word_dict):\n", |
|---|
| 1076 | " sentiment = 0\n", |
|---|
| 1077 | " for token in tokens:\n", |
|---|
| 1078 | " sentiment += score_word_dict.get(token, 0)\n", |
|---|
| 1079 | " return sentiment\n", |
|---|
| 1080 | "df[settings['explanation']][1], get_sentiment_sequence(explanations[1], score_word_dict)" |
|---|
| 1081 | ], |
|---|
| 1082 | "execution_count": null, |
|---|
| 1083 | "outputs": [] |
|---|
| 1084 | }, |
|---|
| 1085 | { |
|---|
| 1086 | "cell_type": "markdown", |
|---|
| 1087 | "metadata": { |
|---|
| 1088 | "id": "eXNNUk16J1zU" |
|---|
| 1089 | }, |
|---|
| 1090 | "source": [ |
|---|
| 1091 | "**OPTIONAL TASK**: Extend the above method, so it is able to explain the sentiment score. For example, the method can return a tuple (`score`, `list of sentiment positive/negative words`)." |
|---|
| 1092 | ] |
|---|
| 1093 | }, |
|---|
| 1094 | { |
|---|
| 1095 | "cell_type": "markdown", |
|---|
| 1096 | "metadata": { |
|---|
| 1097 | "nbpresent": { |
|---|
| 1098 | "id": "69c017ce-9d2f-4e23-9ceb-36e8a09889e0" |
|---|
| 1099 | }, |
|---|
| 1100 | "id": "q7HDBR0xFZyZ" |
|---|
| 1101 | }, |
|---|
| 1102 | "source": [ |
|---|
| 1103 | "Calculate sentiment for all explanations and a column to the DataFrame." |
|---|
| 1104 | ] |
|---|
| 1105 | }, |
|---|
| 1106 | { |
|---|
| 1107 | "cell_type": "code", |
|---|
| 1108 | "metadata": { |
|---|
| 1109 | "nbpresent": { |
|---|
| 1110 | "id": "c7dc10b2-16ff-43df-9402-ce79b38f4d8c" |
|---|
| 1111 | }, |
|---|
| 1112 | "id": "YrVIql33FZyZ" |
|---|
| 1113 | }, |
|---|
| 1114 | "source": [ |
|---|
| 1115 | "scores=[]\n", |
|---|
| 1116 | "for d in explanations:\n", |
|---|
| 1117 | " score = get_sentiment_sequence(d, score_word_dict)\n", |
|---|
| 1118 | " scores.append(score)\n", |
|---|
| 1119 | "df['feeling_score_lexicon'] = scores\n", |
|---|
| 1120 | "df.head()" |
|---|
| 1121 | ], |
|---|
| 1122 | "execution_count": null, |
|---|
| 1123 | "outputs": [] |
|---|
| 1124 | }, |
|---|
| 1125 | { |
|---|
| 1126 | "cell_type": "code", |
|---|
| 1127 | "metadata": { |
|---|
| 1128 | "nbpresent": { |
|---|
| 1129 | "id": "d63764e7-e961-4508-9148-263112f0f5a4" |
|---|
| 1130 | }, |
|---|
| 1131 | "id": "WxGECtObFZyb" |
|---|
| 1132 | }, |
|---|
| 1133 | "source": [ |
|---|
| 1134 | "df.sort_values(by=['feeling_score_lexicon'], ascending=False).head()" |
|---|
| 1135 | ], |
|---|
| 1136 | "execution_count": null, |
|---|
| 1137 | "outputs": [] |
|---|
| 1138 | }, |
|---|
| 1139 | { |
|---|
| 1140 | "cell_type": "markdown", |
|---|
| 1141 | "metadata": { |
|---|
| 1142 | "nbpresent": { |
|---|
| 1143 | "id": "d0a48142-bed2-48de-a84b-219d3139581f" |
|---|
| 1144 | }, |
|---|
| 1145 | "id": "gznwCRXYFZyd" |
|---|
| 1146 | }, |
|---|
| 1147 | "source": [ |
|---|
| 1148 | "## Result 1\n", |
|---|
| 1149 | "\n", |
|---|
| 1150 | "**TASK 2**: Add statistics about the dataset. How many dictionary entries have explanation? How many explanations have sentiment recognized?\n" |
|---|
| 1151 | ] |
|---|
| 1152 | }, |
|---|
| 1153 | { |
|---|
| 1154 | "cell_type": "markdown", |
|---|
| 1155 | "metadata": { |
|---|
| 1156 | "nbpresent": { |
|---|
| 1157 | "id": "ee81f282-5b75-4077-bbf5-c4632d0a2d96" |
|---|
| 1158 | }, |
|---|
| 1159 | "id": "yM0SU5AlFZye" |
|---|
| 1160 | }, |
|---|
| 1161 | "source": [ |
|---|
| 1162 | "## Word Vectors\n", |
|---|
| 1163 | "The main problems of the naive solution are:\n", |
|---|
| 1164 | "* small recall because of rather low quality of the lexicon (due to the automatic translation),\n", |
|---|
| 1165 | "* small recall due to only one form of the word present in the lexicon. However, in Czech language, many different forms for a word exist, e.g. tlustý, tlustým, tlustých, tlustá, tlustého, tlustou.\n", |
|---|
| 1166 | "* no context awareness of the method\n", |
|---|
| 1167 | " \n", |
|---|
| 1168 | " \n", |
|---|
| 1169 | "We try to improve the sentiment recognition using word vectors. The main idea is the *distributional semantics* - an observation that similar words appear in similar contexts. There are many methods how to calculate word vectors, however, all of them take into account not only a token but also tokens in its surrounding (the context). Most techniques use a fixed window, in our case, the window is (-2, +2). For example, for the sentence \"The quick brown fox jumped over the lazy dog.\", using a sliding window we have the following sequences (stopword removal applied in the example):\n", |
|---|
| 1170 | "\n", |
|---|
| 1171 | "\\[quick, brown, fox, jumped, lazy\\]<br>\n", |
|---|
| 1172 | "\\[brown, fox, jumped, lazy, dog\\]\n", |
|---|
| 1173 | "\n", |
|---|
| 1174 | "The distributional semantics assumes that *fox* is similar to other words that appear around the words *quick*, *brown*, *jumped*, *lazy*, *dog*." |
|---|
| 1175 | ] |
|---|
| 1176 | }, |
|---|
| 1177 | { |
|---|
| 1178 | "cell_type": "markdown", |
|---|
| 1179 | "metadata": { |
|---|
| 1180 | "nbpresent": { |
|---|
| 1181 | "id": "8950d61e-500f-4004-917a-6613dec78eac" |
|---|
| 1182 | }, |
|---|
| 1183 | "id": "ROcE6_gpFZye" |
|---|
| 1184 | }, |
|---|
| 1185 | "source": [ |
|---|
| 1186 | "### Token index\n", |
|---|
| 1187 | "We convert the tokens in explanations (without stopwords) into an index. This is handy, since we will only calculate with numbers and provide the respective tokens via this token index only." |
|---|
| 1188 | ] |
|---|
| 1189 | }, |
|---|
| 1190 | { |
|---|
| 1191 | "cell_type": "code", |
|---|
| 1192 | "metadata": { |
|---|
| 1193 | "nbpresent": { |
|---|
| 1194 | "id": "c5c51550-7869-4d63-a72e-73a3305c82f7" |
|---|
| 1195 | }, |
|---|
| 1196 | "id": "1QGKLtu4FZyf" |
|---|
| 1197 | }, |
|---|
| 1198 | "source": [ |
|---|
| 1199 | "tok2indx = dict()\n", |
|---|
| 1200 | "unigram_counts = Counter()\n", |
|---|
| 1201 | "for ii, explanation in enumerate(explanations):\n", |
|---|
| 1202 | " for token in explanation:\n", |
|---|
| 1203 | " unigram_counts[token] += 1\n", |
|---|
| 1204 | " if token not in tok2indx:\n", |
|---|
| 1205 | " tok2indx[token] = len(tok2indx)\n", |
|---|
| 1206 | "indx2tok = {indx:tok for tok,indx in tok2indx.items()}\n", |
|---|
| 1207 | "print('done')\n", |
|---|
| 1208 | "print('vocabulary size: {}'.format(len(unigram_counts)))\n", |
|---|
| 1209 | "print('most common: {}'.format(unigram_counts.most_common(10)))" |
|---|
| 1210 | ], |
|---|
| 1211 | "execution_count": null, |
|---|
| 1212 | "outputs": [] |
|---|
| 1213 | }, |
|---|
| 1214 | { |
|---|
| 1215 | "cell_type": "markdown", |
|---|
| 1216 | "metadata": { |
|---|
| 1217 | "nbpresent": { |
|---|
| 1218 | "id": "122732e5-5b39-4ba8-9fbf-6834c9f6b085" |
|---|
| 1219 | }, |
|---|
| 1220 | "id": "njoy0tQ-FZyh" |
|---|
| 1221 | }, |
|---|
| 1222 | "source": [ |
|---|
| 1223 | "### Skipgrams\n", |
|---|
| 1224 | "We calculate the frequencies of word tuples appearing in the same sliding window. You can see the most frequent tuples." |
|---|
| 1225 | ] |
|---|
| 1226 | }, |
|---|
| 1227 | { |
|---|
| 1228 | "cell_type": "code", |
|---|
| 1229 | "metadata": { |
|---|
| 1230 | "nbpresent": { |
|---|
| 1231 | "id": "86272d04-2637-4206-885b-4873f4b50a80" |
|---|
| 1232 | }, |
|---|
| 1233 | "id": "Pdph1BsTFZyh" |
|---|
| 1234 | }, |
|---|
| 1235 | "source": [ |
|---|
| 1236 | "back_window = 2\n", |
|---|
| 1237 | "front_window = 2\n", |
|---|
| 1238 | "skipgram_counts = Counter()\n", |
|---|
| 1239 | "for i, explanation in enumerate(explanations):\n", |
|---|
| 1240 | " for ifw, fw in enumerate(explanation):\n", |
|---|
| 1241 | " icw_min = max(0, ifw - back_window)\n", |
|---|
| 1242 | " icw_max = min(len(explanation) - 1, ifw + front_window)\n", |
|---|
| 1243 | " icws = [ii for ii in range(icw_min, icw_max + 1) if ii != ifw]\n", |
|---|
| 1244 | " for icw in icws:\n", |
|---|
| 1245 | " skipgram = (explanation[ifw], explanation[icw])\n", |
|---|
| 1246 | " skipgram_counts[skipgram] += 1 \n", |
|---|
| 1247 | " \n", |
|---|
| 1248 | "print('done')\n", |
|---|
| 1249 | "print('number of skipgrams: {}'.format(len(skipgram_counts)))\n", |
|---|
| 1250 | "print('most common: {}'.format(skipgram_counts.most_common(10)))" |
|---|
| 1251 | ], |
|---|
| 1252 | "execution_count": null, |
|---|
| 1253 | "outputs": [] |
|---|
| 1254 | }, |
|---|
| 1255 | { |
|---|
| 1256 | "cell_type": "markdown", |
|---|
| 1257 | "metadata": { |
|---|
| 1258 | "nbpresent": { |
|---|
| 1259 | "id": "d233effc-9d8e-40df-9765-b53dfdff3c68" |
|---|
| 1260 | }, |
|---|
| 1261 | "id": "HUNFAWM2FZyj" |
|---|
| 1262 | }, |
|---|
| 1263 | "source": [ |
|---|
| 1264 | "## Token matrix\n", |
|---|
| 1265 | "We store the skipgram frequencies in a (symmetric) matrix.\n", |
|---|
| 1266 | "\n", |
|---|
| 1267 | "**OPTIONAL TASK**: Why is the matrix symmetric?" |
|---|
| 1268 | ] |
|---|
| 1269 | }, |
|---|
| 1270 | { |
|---|
| 1271 | "cell_type": "code", |
|---|
| 1272 | "metadata": { |
|---|
| 1273 | "nbpresent": { |
|---|
| 1274 | "id": "9dda77d1-4b69-48c2-b603-750748d4af25" |
|---|
| 1275 | }, |
|---|
| 1276 | "id": "n7RvFvKSFZyk" |
|---|
| 1277 | }, |
|---|
| 1278 | "source": [ |
|---|
| 1279 | "row_indxs = []\n", |
|---|
| 1280 | "col_indxs = []\n", |
|---|
| 1281 | "dat_values = []\n", |
|---|
| 1282 | "ii = 0\n", |
|---|
| 1283 | "for (tok1, tok2), sg_count in skipgram_counts.items():\n", |
|---|
| 1284 | " ii += 1\n", |
|---|
| 1285 | " if ii % 1000000 == 0:\n", |
|---|
| 1286 | " print(f'finished {ii/len(skipgram_counts):.2%} of skipgrams')\n", |
|---|
| 1287 | " tok1_indx = tok2indx[tok1]\n", |
|---|
| 1288 | " tok2_indx = tok2indx[tok2]\n", |
|---|
| 1289 | " \n", |
|---|
| 1290 | " row_indxs.append(tok1_indx)\n", |
|---|
| 1291 | " col_indxs.append(tok2_indx)\n", |
|---|
| 1292 | " dat_values.append(sg_count)\n", |
|---|
| 1293 | " \n", |
|---|
| 1294 | "wwcnt_mat = sparse.csr_matrix((dat_values, (row_indxs, col_indxs)))\n", |
|---|
| 1295 | "print(wwwcnt_mat.shape)" |
|---|
| 1296 | ], |
|---|
| 1297 | "execution_count": null, |
|---|
| 1298 | "outputs": [] |
|---|
| 1299 | }, |
|---|
| 1300 | { |
|---|
| 1301 | "cell_type": "markdown", |
|---|
| 1302 | "metadata": { |
|---|
| 1303 | "nbpresent": { |
|---|
| 1304 | "id": "7a8821c7-2bd8-4330-a54e-48f61b0516b7" |
|---|
| 1305 | }, |
|---|
| 1306 | "id": "soA0rDYAFZyl" |
|---|
| 1307 | }, |
|---|
| 1308 | "source": [ |
|---|
| 1309 | "## Token similarity\n", |
|---|
| 1310 | "In the token matrix, each row correspond to a token, the row is the word vector. The numbers in tha matrix show how often the token appears together with other tokens.\n", |
|---|
| 1311 | "The following method calculates token similarity as a cosine of the angle between two word vectors." |
|---|
| 1312 | ] |
|---|
| 1313 | }, |
|---|
| 1314 | { |
|---|
| 1315 | "cell_type": "code", |
|---|
| 1316 | "metadata": { |
|---|
| 1317 | "nbpresent": { |
|---|
| 1318 | "id": "f06795b7-b03f-4954-9ca6-3a1fb19fa500" |
|---|
| 1319 | }, |
|---|
| 1320 | "id": "QS5uTID7FZym" |
|---|
| 1321 | }, |
|---|
| 1322 | "source": [ |
|---|
| 1323 | "def ww_sim(token, matrix, topn=10):\n", |
|---|
| 1324 | " \"\"\"Calculate topn most similar words to word\"\"\"\n", |
|---|
| 1325 | " if token not in tok2indx:\n", |
|---|
| 1326 | " return 0\n", |
|---|
| 1327 | " indx = tok2indx[token]\n", |
|---|
| 1328 | " if isinstance(matrix, sparse.csr_matrix):\n", |
|---|
| 1329 | " v1 = matrix.getrow(indx)\n", |
|---|
| 1330 | " else:\n", |
|---|
| 1331 | " v1 = matrix[indx:indx+1, :]\n", |
|---|
| 1332 | " sims = cosine_similarity(matrix, v1).flatten()\n", |
|---|
| 1333 | " sindxs = np.argsort(-sims)\n", |
|---|
| 1334 | " sim_word_scores = [(indx2tok[sindx], sims[sindx]) for sindx in sindxs[0:topn]]\n", |
|---|
| 1335 | " return sim_word_scores" |
|---|
| 1336 | ], |
|---|
| 1337 | "execution_count": null, |
|---|
| 1338 | "outputs": [] |
|---|
| 1339 | }, |
|---|
| 1340 | { |
|---|
| 1341 | "cell_type": "code", |
|---|
| 1342 | "metadata": { |
|---|
| 1343 | "nbpresent": { |
|---|
| 1344 | "id": "de5b327b-9089-482f-ae6f-39e6fe3dac32" |
|---|
| 1345 | }, |
|---|
| 1346 | "id": "_5olMNOXFZyo" |
|---|
| 1347 | }, |
|---|
| 1348 | "source": [ |
|---|
| 1349 | "import pprint\n", |
|---|
| 1350 | "if selected_language == \"cs\":\n", |
|---|
| 1351 | " pprint.pprint(ww_sim('obézní', wwcnt_mat))\n", |
|---|
| 1352 | " pprint.pprint(ww_sim('hezká', wwcnt_mat))\n", |
|---|
| 1353 | "else:\n", |
|---|
| 1354 | " pprint.pprint(ww_sim('ugly', wwcnt_mat))\n", |
|---|
| 1355 | " pprint.pprint(ww_sim('girl', wwcnt_mat))" |
|---|
| 1356 | ], |
|---|
| 1357 | "execution_count": null, |
|---|
| 1358 | "outputs": [] |
|---|
| 1359 | }, |
|---|
| 1360 | { |
|---|
| 1361 | "cell_type": "markdown", |
|---|
| 1362 | "metadata": { |
|---|
| 1363 | "nbpresent": { |
|---|
| 1364 | "id": "5c48f967-8d12-43d2-87e8-a08e5ebfb660" |
|---|
| 1365 | }, |
|---|
| 1366 | "id": "IGBk7RkEFZyr" |
|---|
| 1367 | }, |
|---|
| 1368 | "source": [ |
|---|
| 1369 | "## Calculate sentiment for OOVs\n", |
|---|
| 1370 | "Out-of-vocabulary terms is one of the problems in our first try. Let's try to expand the Opinion Lexicon by similar words. You can see that we now know the sentiment of words *not* present in the Opinion Lexicon." |
|---|
| 1371 | ] |
|---|
| 1372 | }, |
|---|
| 1373 | { |
|---|
| 1374 | "cell_type": "code", |
|---|
| 1375 | "metadata": { |
|---|
| 1376 | "nbpresent": { |
|---|
| 1377 | "id": "cf2caeae-95b8-45d3-8f78-6293cf823920" |
|---|
| 1378 | }, |
|---|
| 1379 | "id": "b8W42bq3FZyr" |
|---|
| 1380 | }, |
|---|
| 1381 | "source": [ |
|---|
| 1382 | "def get_sentiment(token, score_word_dict, ww_matrix):\n", |
|---|
| 1383 | "# if token in score_word_dict.keys(): # alternative: calculate from vectors iff the word is not in the lexicon\n", |
|---|
| 1384 | "# return score_word_dict[token]\n", |
|---|
| 1385 | " sentiment = score_word_dict.get(token, 0)\n", |
|---|
| 1386 | " k = ww_sim(token, ww_matrix)\n", |
|---|
| 1387 | " if not k:\n", |
|---|
| 1388 | " return 0\n", |
|---|
| 1389 | " for sim, score in k:\n", |
|---|
| 1390 | " sentiment += score * score_word_dict.get(sim, 0)\n", |
|---|
| 1391 | " return sentiment\n", |
|---|
| 1392 | "if selected_language==\"cs\":\n", |
|---|
| 1393 | " print('andrej', get_sentiment('andrej', score_word_dict, wwcnt_mat), score_word_dict.get('andrej'))\n", |
|---|
| 1394 | "else:\n", |
|---|
| 1395 | " print('donald', get_sentiment('donald', score_word_dict, wwcnt_mat), score_word_dict.get('donald'))" |
|---|
| 1396 | ], |
|---|
| 1397 | "execution_count": null, |
|---|
| 1398 | "outputs": [] |
|---|
| 1399 | }, |
|---|
| 1400 | { |
|---|
| 1401 | "cell_type": "code", |
|---|
| 1402 | "metadata": { |
|---|
| 1403 | "nbpresent": { |
|---|
| 1404 | "id": "8a29bb08-cd2f-4c83-b858-15c44d56030e" |
|---|
| 1405 | }, |
|---|
| 1406 | "id": "cVvAhOmPFZyt" |
|---|
| 1407 | }, |
|---|
| 1408 | "source": [ |
|---|
| 1409 | "def get_sentiment_sequence(tokens, score_word_dict, ww_matrix):\n", |
|---|
| 1410 | " sentiment = 0\n", |
|---|
| 1411 | " for token in tokens:\n", |
|---|
| 1412 | " sentiment += get_sentiment(token, score_word_dict, ww_matrix)\n", |
|---|
| 1413 | " return sentiment\n", |
|---|
| 1414 | "df[settings['explanation']][1], get_sentiment_sequence(explanations[1], score_word_dict, wwcnt_mat)" |
|---|
| 1415 | ], |
|---|
| 1416 | "execution_count": null, |
|---|
| 1417 | "outputs": [] |
|---|
| 1418 | }, |
|---|
| 1419 | { |
|---|
| 1420 | "cell_type": "code", |
|---|
| 1421 | "metadata": { |
|---|
| 1422 | "nbpresent": { |
|---|
| 1423 | "id": "c0a1ebc8-217f-4d49-9bcd-eeca4debf726" |
|---|
| 1424 | }, |
|---|
| 1425 | "id": "Unf6elzYFZyv" |
|---|
| 1426 | }, |
|---|
| 1427 | "source": [ |
|---|
| 1428 | "# This operation takes time. It calculates sentiment for each word in all explanations from the matrix. It could be optimized e.g. by not calculating the same word multiple times.\n", |
|---|
| 1429 | "# We calculate the time spent using the time magic.\n", |
|---|
| 1430 | "def calculate_scores(explanations, score_word_dict, wwcnt_mat):\n", |
|---|
| 1431 | " scores=[]\n", |
|---|
| 1432 | " for d in explanations:\n", |
|---|
| 1433 | " score = get_sentiment_sequence(d, score_word_dict, wwcnt_mat)\n", |
|---|
| 1434 | " scores.append(score)\n", |
|---|
| 1435 | " return scores\n", |
|---|
| 1436 | "%time scores = calculate_scores(explanations, score_word_dict, wwcnt_mat)\n", |
|---|
| 1437 | "df['feeling_score_wv'] = scores\n", |
|---|
| 1438 | "df.head()" |
|---|
| 1439 | ], |
|---|
| 1440 | "execution_count": null, |
|---|
| 1441 | "outputs": [] |
|---|
| 1442 | }, |
|---|
| 1443 | { |
|---|
| 1444 | "cell_type": "code", |
|---|
| 1445 | "metadata": { |
|---|
| 1446 | "nbpresent": { |
|---|
| 1447 | "id": "61acf6bd-aeec-49fb-874a-ec19691ee757" |
|---|
| 1448 | }, |
|---|
| 1449 | "id": "11kQAobgFZyx" |
|---|
| 1450 | }, |
|---|
| 1451 | "source": [ |
|---|
| 1452 | "df.sort_values(by=['feeling_score_wv']).head()" |
|---|
| 1453 | ], |
|---|
| 1454 | "execution_count": null, |
|---|
| 1455 | "outputs": [] |
|---|
| 1456 | }, |
|---|
| 1457 | { |
|---|
| 1458 | "cell_type": "markdown", |
|---|
| 1459 | "metadata": { |
|---|
| 1460 | "nbpresent": { |
|---|
| 1461 | "id": "9fcfcdbd-7424-4cdc-a7be-23d241c7a20a" |
|---|
| 1462 | }, |
|---|
| 1463 | "id": "FMdtb7f_FZyy" |
|---|
| 1464 | }, |
|---|
| 1465 | "source": [ |
|---|
| 1466 | "## Result 2\n", |
|---|
| 1467 | "\n", |
|---|
| 1468 | "**TASK 3**: Calculate the same statistics as for Result 1. How did word vectors improve the number of recognized words in the explanations?" |
|---|
| 1469 | ] |
|---|
| 1470 | }, |
|---|
| 1471 | { |
|---|
| 1472 | "cell_type": "markdown", |
|---|
| 1473 | "metadata": { |
|---|
| 1474 | "nbpresent": { |
|---|
| 1475 | "id": "6e503bce-561b-49d9-883d-8b93c8f176ca" |
|---|
| 1476 | }, |
|---|
| 1477 | "id": "L_wOHgRWFZyz" |
|---|
| 1478 | }, |
|---|
| 1479 | "source": [ |
|---|
| 1480 | "## Evaluation\n", |
|---|
| 1481 | "For this course, we manually annotated sentiment for 400 Czech explanations. We will compare the sentiment recognized by our methods with the manual annotation." |
|---|
| 1482 | ] |
|---|
| 1483 | }, |
|---|
| 1484 | { |
|---|
| 1485 | "cell_type": "code", |
|---|
| 1486 | "metadata": { |
|---|
| 1487 | "id": "ruRLnhGrNQng" |
|---|
| 1488 | }, |
|---|
| 1489 | "source": [ |
|---|
| 1490 | "!wget http://nlp.fi.muni.cz/trac/research/raw-attachment/wiki/private/AdvancedNlpCourse/OpinionSentiment/cestina20_annotation.csv" |
|---|
| 1491 | ], |
|---|
| 1492 | "execution_count": null, |
|---|
| 1493 | "outputs": [] |
|---|
| 1494 | }, |
|---|
| 1495 | { |
|---|
| 1496 | "cell_type": "code", |
|---|
| 1497 | "metadata": { |
|---|
| 1498 | "nbpresent": { |
|---|
| 1499 | "id": "9874742a-f8f7-4cf9-8f3c-a040b47c41d9" |
|---|
| 1500 | }, |
|---|
| 1501 | "id": "mmpWdKetFZyz" |
|---|
| 1502 | }, |
|---|
| 1503 | "source": [ |
|---|
| 1504 | "annotation = None\n", |
|---|
| 1505 | "if selected_language==\"cs\":\n", |
|---|
| 1506 | " dfa = pd.read_csv('cestina20_annotation.csv', sep=';')\n", |
|---|
| 1507 | " annotation = pd.merge(df, dfa, on='word') #df.loc[:len(dfa)-1,]\n", |
|---|
| 1508 | "else:\n", |
|---|
| 1509 | " print(\"no ground truth (manual annotations) available\")" |
|---|
| 1510 | ], |
|---|
| 1511 | "execution_count": null, |
|---|
| 1512 | "outputs": [] |
|---|
| 1513 | }, |
|---|
| 1514 | { |
|---|
| 1515 | "cell_type": "code", |
|---|
| 1516 | "metadata": { |
|---|
| 1517 | "id": "OuyXEUseh9FE" |
|---|
| 1518 | }, |
|---|
| 1519 | "source": [ |
|---|
| 1520 | "annotation.head()" |
|---|
| 1521 | ], |
|---|
| 1522 | "execution_count": null, |
|---|
| 1523 | "outputs": [] |
|---|
| 1524 | }, |
|---|
| 1525 | { |
|---|
| 1526 | "cell_type": "markdown", |
|---|
| 1527 | "metadata": { |
|---|
| 1528 | "nbpresent": { |
|---|
| 1529 | "id": "f5a8fe74-4b1e-4ad3-b45b-344ae29f540e" |
|---|
| 1530 | }, |
|---|
| 1531 | "id": "vh-YUf6yFZy1" |
|---|
| 1532 | }, |
|---|
| 1533 | "source": [ |
|---|
| 1534 | "**TASK 4 (cs)**: Insert code to calculate confusion matrix for both `feeling_score_lexicon` and `feeling_score_wv` with `annotation`. Assume that the opinion recognition is correct if the score == 0 or the same polarity. Which sentiment recognition is more accurate? Why?\n", |
|---|
| 1535 | "\n", |
|---|
| 1536 | "**TASK 4 (en)**: Go through 100 classifications and try to annotate manually the sentiment, -1 for negative, 0 for neutral, and 1 for positive is enough, i.e. do not consider *intensity*, only *polarity*." |
|---|
| 1537 | ] |
|---|
| 1538 | }, |
|---|
| 1539 | { |
|---|
| 1540 | "cell_type": "code", |
|---|
| 1541 | "metadata": { |
|---|
| 1542 | "nbpresent": { |
|---|
| 1543 | "id": "332417aa-251e-4477-bac7-59e34406c66e" |
|---|
| 1544 | }, |
|---|
| 1545 | "id": "Sm0Nc8rgFZy2" |
|---|
| 1546 | }, |
|---|
| 1547 | "source": [ |
|---|
| 1548 | "" |
|---|
| 1549 | ], |
|---|
| 1550 | "execution_count": null, |
|---|
| 1551 | "outputs": [] |
|---|
| 1552 | } |
|---|
| 1553 | ] |
|---|
| 1554 | } |
|---|
