1/*-
  2 * SPDX-License-Identifier: BSD-3-Clause
  3 *
  4 * Copyright (c) 1993 The Regents of the University of California.
  5 * All rights reserved.
  6 *
  7 * Redistribution and use in source and binary forms, with or without
  8 * modification, are permitted provided that the following conditions
  9 * are met:
 10 * 1. Redistributions of source code must retain the above copyright
 11 *    notice, this list of conditions and the following disclaimer.
 12 * 2. Redistributions in binary form must reproduce the above copyright
 13 *    notice, this list of conditions and the following disclaimer in the
 14 *    documentation and/or other materials provided with the distribution.
 15 * 3. Neither the name of the University nor the names of its contributors
 16 *    may be used to endorse or promote products derived from this software
 17 *    without specific prior written permission.
 18 *
 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 22 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 29 * SUCH DAMAGE.
 30 */
 31
 32/*
 33 * Functions to provide access to special i386 instructions.
 34 * This in included in sys/systm.h, and that file should be
 35 * used in preference to this.
 36 */
 37
 38#ifndef _MACHINE_CPUFUNC_H_
 39#define	_MACHINE_CPUFUNC_H_
 40
 41struct region_descriptor;
 42
 43#define readb(va)	(*(volatile uint8_t *) (va))
 44#define readw(va)	(*(volatile uint16_t *) (va))
 45#define readl(va)	(*(volatile uint32_t *) (va))
 46
 47#define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
 48#define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
 49#define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
 50
 51static __inline void
 52breakpoint(void)
 53{
 54	__asm __volatile("int $3");
 55}
 56
 57static __inline __pure2 u_int
 58bsfl(u_int mask)
 59{
 60	u_int	result;
 61
 62	__asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
 63	return (result);
 64}
 65
 66static __inline __pure2 u_int
 67bsrl(u_int mask)
 68{
 69	u_int	result;
 70
 71	__asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
 72	return (result);
 73}
 74
 75static __inline void
 76clflush(u_long addr)
 77{
 78
 79	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
 80}
 81
 82static __inline void
 83clflushopt(u_long addr)
 84{
 85
 86	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
 87}
 88
 89static __inline void
 90clts(void)
 91{
 92
 93	__asm __volatile("clts");
 94}
 95
 96static __inline void
 97disable_intr(void)
 98{
 99	__asm __volatile("cli" : : : "memory");
100}
101
102#ifdef _KERNEL
103static __inline void
104do_cpuid(u_int ax, u_int *p)
105{
106	__asm __volatile("cpuid"
107	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
108	    :  "0" (ax));
109}
110
111static __inline void
112cpuid_count(u_int ax, u_int cx, u_int *p)
113{
114	__asm __volatile("cpuid"
115	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
116	    :  "0" (ax), "c" (cx));
117}
118#else
119static __inline void
120do_cpuid(u_int ax, u_int *p)
121{
122	__asm __volatile(
123	    "pushl\t%%ebx\n\t"
124	    "cpuid\n\t"
125	    "movl\t%%ebx,%1\n\t"
126	    "popl\t%%ebx"
127	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
128	    :  "0" (ax));
129}
130
131static __inline void
132cpuid_count(u_int ax, u_int cx, u_int *p)
133{
134	__asm __volatile(
135	    "pushl\t%%ebx\n\t"
136	    "cpuid\n\t"
137	    "movl\t%%ebx,%1\n\t"
138	    "popl\t%%ebx"
139	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
140	    :  "0" (ax), "c" (cx));
141}
142#endif
143
144static __inline void
145enable_intr(void)
146{
147	__asm __volatile("sti");
148}
149
150static __inline void
151cpu_monitor(const void *addr, u_long extensions, u_int hints)
152{
153	__asm __volatile("monitor"
154	    : : "a" (addr), "c" (extensions), "d" (hints));
155}
156
157static __inline void
158cpu_mwait(u_long extensions, u_int hints)
159{
160	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
161}
162
163static __inline void
164lfence(void)
165{
166	__asm __volatile("lfence" : : : "memory");
167}
168
169static __inline void
170mfence(void)
171{
172	__asm __volatile("mfence" : : : "memory");
173}
174
175static __inline void
176sfence(void)
177{
178	__asm __volatile("sfence" : : : "memory");
179}
180
181static __inline void
182halt(void)
183{
184	__asm __volatile("hlt");
185}
186
187static __inline u_char
188inb(u_int port)
189{
190	u_char	data;
191
192	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
193	return (data);
194}
195
196static __inline u_int
197inl(u_int port)
198{
199	u_int	data;
200
201	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
202	return (data);
203}
204
205static __inline void
206insb(u_int port, void *addr, size_t count)
207{
208	__asm __volatile("cld; rep; insb"
209			 : "+D" (addr), "+c" (count)
210			 : "d" (port)
211			 : "memory");
212}
213
214static __inline void
215insw(u_int port, void *addr, size_t count)
216{
217	__asm __volatile("cld; rep; insw"
218			 : "+D" (addr), "+c" (count)
219			 : "d" (port)
220			 : "memory");
221}
222
223static __inline void
224insl(u_int port, void *addr, size_t count)
225{
226	__asm __volatile("cld; rep; insl"
227			 : "+D" (addr), "+c" (count)
228			 : "d" (port)
229			 : "memory");
230}
231
232static __inline void
233invd(void)
234{
235	__asm __volatile("invd");
236}
237
238static __inline u_short
239inw(u_int port)
240{
241	u_short	data;
242
243	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
244	return (data);
245}
246
247static __inline void
248outb(u_int port, u_char data)
249{
250	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
251}
252
253static __inline void
254outl(u_int port, u_int data)
255{
256	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
257}
258
259static __inline void
260outsb(u_int port, const void *addr, size_t count)
261{
262	__asm __volatile("cld; rep; outsb"
263			 : "+S" (addr), "+c" (count)
264			 : "d" (port));
265}
266
267static __inline void
268outsw(u_int port, const void *addr, size_t count)
269{
270	__asm __volatile("cld; rep; outsw"
271			 : "+S" (addr), "+c" (count)
272			 : "d" (port));
273}
274
275static __inline void
276outsl(u_int port, const void *addr, size_t count)
277{
278	__asm __volatile("cld; rep; outsl"
279			 : "+S" (addr), "+c" (count)
280			 : "d" (port));
281}
282
283static __inline void
284outw(u_int port, u_short data)
285{
286	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
287}
288
289static __inline void
290ia32_pause(void)
291{
292	__asm __volatile("pause");
293}
294
295static __inline u_int
296read_eflags(void)
297{
298	u_int	ef;
299
300	__asm __volatile("pushfl; popl %0" : "=r" (ef));
301	return (ef);
302}
303
304static __inline uint64_t
305rdmsr(u_int msr)
306{
307	uint64_t rv;
308
309	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
310	return (rv);
311}
312
313static __inline uint32_t
314rdmsr32(u_int msr)
315{
316	uint32_t low;
317
318	__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
319	return (low);
320}
321
322static __inline uint64_t
323rdpmc(u_int pmc)
324{
325	uint64_t rv;
326
327	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
328	return (rv);
329}
330
331static __inline uint64_t
332rdtsc(void)
333{
334	uint64_t rv;
335
336	__asm __volatile("rdtsc" : "=A" (rv));
337	return (rv);
338}
339
340static __inline uint64_t
341rdtsc_ordered_lfence(void)
342{
343	lfence();
344	return (rdtsc());
345}
346
347static __inline uint64_t
348rdtsc_ordered_mfence(void)
349{
350	mfence();
351	return (rdtsc());
352}
353
354static __inline uint64_t
355rdtscp(void)
356{
357	uint64_t rv;
358
359	__asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
360	return (rv);
361}
362
363static __inline uint64_t
364rdtscp_aux(uint32_t *aux)
365{
366	uint64_t rv;
367
368	__asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux));
369	return (rv);
370}
371
372static __inline uint32_t
373rdtsc32(void)
374{
375	uint32_t rv;
376
377	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
378	return (rv);
379}
380
381static __inline uint32_t
382rdtscp32(void)
383{
384	uint32_t rv;
385
386	__asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
387	return (rv);
388}
389
390static __inline void
391wbinvd(void)
392{
393	__asm __volatile("wbinvd");
394}
395
396static __inline void
397write_eflags(u_int ef)
398{
399	__asm __volatile("pushl %0; popfl" : : "r" (ef));
400}
401
402static __inline void
403wrmsr(u_int msr, uint64_t newval)
404{
405	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
406}
407
408static __inline void
409load_cr0(u_int data)
410{
411
412	__asm __volatile("movl %0,%%cr0" : : "r" (data));
413}
414
415static __inline u_int
416rcr0(void)
417{
418	u_int	data;
419
420	__asm __volatile("movl %%cr0,%0" : "=r" (data));
421	return (data);
422}
423
424static __inline u_int
425rcr2(void)
426{
427	u_int	data;
428
429	__asm __volatile("movl %%cr2,%0" : "=r" (data));
430	return (data);
431}
432
433static __inline void
434load_cr3(u_int data)
435{
436
437	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
438}
439
440static __inline u_int
441rcr3(void)
442{
443	u_int	data;
444
445	__asm __volatile("movl %%cr3,%0" : "=r" (data));
446	return (data);
447}
448
449static __inline void
450load_cr4(u_int data)
451{
452	__asm __volatile("movl %0,%%cr4" : : "r" (data));
453}
454
455static __inline u_int
456rcr4(void)
457{
458	u_int	data;
459
460	__asm __volatile("movl %%cr4,%0" : "=r" (data));
461	return (data);
462}
463
464static __inline uint64_t
465rxcr(u_int reg)
466{
467	u_int low, high;
468
469	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
470	return (low | ((uint64_t)high << 32));
471}
472
473static __inline void
474load_xcr(u_int reg, uint64_t val)
475{
476	u_int low, high;
477
478	low = val;
479	high = val >> 32;
480	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
481}
482
483/*
484 * Global TLB flush (except for thise for pages marked PG_G)
485 */
486static __inline void
487invltlb(void)
488{
489
490	load_cr3(rcr3());
491}
492
493/*
494 * TLB flush for an individual page (even if it has PG_G).
495 * Only works on 486+ CPUs (i386 does not have PG_G).
496 */
497static __inline void
498invlpg(u_int addr)
499{
500
501	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
502}
503
504static __inline u_short
505rfs(void)
506{
507	u_short sel;
508	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
509	return (sel);
510}
511
512static __inline uint64_t
513rgdt(void)
514{
515	uint64_t gdtr;
516	__asm __volatile("sgdt %0" : "=m" (gdtr));
517	return (gdtr);
518}
519
520static __inline u_short
521rgs(void)
522{
523	u_short sel;
524	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
525	return (sel);
526}
527
528static __inline uint64_t
529ridt(void)
530{
531	uint64_t idtr;
532	__asm __volatile("sidt %0" : "=m" (idtr));
533	return (idtr);
534}
535
536static __inline u_short
537rldt(void)
538{
539	u_short ldtr;
540	__asm __volatile("sldt %0" : "=g" (ldtr));
541	return (ldtr);
542}
543
544static __inline u_short
545rss(void)
546{
547	u_short sel;
548	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
549	return (sel);
550}
551
552static __inline u_short
553rtr(void)
554{
555	u_short tr;
556	__asm __volatile("str %0" : "=g" (tr));
557	return (tr);
558}
559
560static __inline void
561load_fs(u_short sel)
562{
563	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
564}
565
566static __inline void
567load_gs(u_short sel)
568{
569	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
570}
571
572static __inline void
573lidt(struct region_descriptor *addr)
574{
575	__asm __volatile("lidt (%0)" : : "r" (addr));
576}
577
578static __inline void
579lldt(u_short sel)
580{
581	__asm __volatile("lldt %0" : : "r" (sel));
582}
583
584static __inline void
585ltr(u_short sel)
586{
587	__asm __volatile("ltr %0" : : "r" (sel));
588}
589
590static __inline u_int
591rdr0(void)
592{
593	u_int	data;
594	__asm __volatile("movl %%dr0,%0" : "=r" (data));
595	return (data);
596}
597
598static __inline void
599load_dr0(u_int dr0)
600{
601	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
602}
603
604static __inline u_int
605rdr1(void)
606{
607	u_int	data;
608	__asm __volatile("movl %%dr1,%0" : "=r" (data));
609	return (data);
610}
611
612static __inline void
613load_dr1(u_int dr1)
614{
615	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
616}
617
618static __inline u_int
619rdr2(void)
620{
621	u_int	data;
622	__asm __volatile("movl %%dr2,%0" : "=r" (data));
623	return (data);
624}
625
626static __inline void
627load_dr2(u_int dr2)
628{
629	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
630}
631
632static __inline u_int
633rdr3(void)
634{
635	u_int	data;
636	__asm __volatile("movl %%dr3,%0" : "=r" (data));
637	return (data);
638}
639
640static __inline void
641load_dr3(u_int dr3)
642{
643	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
644}
645
646static __inline u_int
647rdr6(void)
648{
649	u_int	data;
650	__asm __volatile("movl %%dr6,%0" : "=r" (data));
651	return (data);
652}
653
654static __inline void
655load_dr6(u_int dr6)
656{
657	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
658}
659
660static __inline u_int
661rdr7(void)
662{
663	u_int	data;
664	__asm __volatile("movl %%dr7,%0" : "=r" (data));
665	return (data);
666}
667
668static __inline void
669load_dr7(u_int dr7)
670{
671	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
672}
673
674static __inline u_char
675read_cyrix_reg(u_char reg)
676{
677	outb(0x22, reg);
678	return inb(0x23);
679}
680
681static __inline void
682write_cyrix_reg(u_char reg, u_char data)
683{
684	outb(0x22, reg);
685	outb(0x23, data);
686}
687
688static __inline register_t
689intr_disable(void)
690{
691	register_t eflags;
692
693	eflags = read_eflags();
694	disable_intr();
695	return (eflags);
696}
697
698static __inline void
699intr_restore(register_t eflags)
700{
701	write_eflags(eflags);
702}
703
704static __inline uint32_t
705rdpkru(void)
706{
707	uint32_t res;
708
709	__asm __volatile("rdpkru" :  "=a" (res) : "c" (0) : "edx");
710	return (res);
711}
712
713static __inline void
714wrpkru(uint32_t mask)
715{
716
717	__asm __volatile("wrpkru" :  : "a" (mask),  "c" (0), "d" (0));
718}
719
720void    reset_dbregs(void);
721
722#ifdef _KERNEL
723int	rdmsr_safe(u_int msr, uint64_t *val);
724int	wrmsr_safe(u_int msr, uint64_t newval);
725#endif
726
727#endif /* !_MACHINE_CPUFUNC_H_ */